2024-12-06 14:41:34,532 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-06 14:41:34,545 main DEBUG Took 0.010351 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 14:41:34,545 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 14:41:34,545 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 14:41:34,546 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 14:41:34,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,555 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 14:41:34,567 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,569 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,570 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,571 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,571 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,571 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,572 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,572 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,573 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,574 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,574 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,575 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,575 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,575 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,575 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,576 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,576 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,577 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,577 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:41:34,577 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,577 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 14:41:34,579 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:41:34,581 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 14:41:34,582 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 14:41:34,583 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 14:41:34,584 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 14:41:34,584 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 14:41:34,592 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 14:41:34,594 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 14:41:34,596 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 14:41:34,596 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 14:41:34,597 main DEBUG createAppenders(={Console}) 2024-12-06 14:41:34,597 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-06 14:41:34,598 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-06 14:41:34,598 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-06 14:41:34,598 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 14:41:34,599 main DEBUG OutputStream closed 2024-12-06 14:41:34,599 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 14:41:34,599 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 14:41:34,599 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-06 14:41:34,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 14:41:34,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 14:41:34,669 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 14:41:34,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 14:41:34,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 14:41:34,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 14:41:34,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 14:41:34,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 14:41:34,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 14:41:34,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 14:41:34,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 14:41:34,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 14:41:34,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 14:41:34,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 14:41:34,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 14:41:34,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 14:41:34,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 14:41:34,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 14:41:34,680 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 14:41:34,680 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-06 14:41:34,681 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 14:41:34,682 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-06T14:41:34,947 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520 2024-12-06 14:41:34,950 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 14:41:34,950 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T14:41:34,959 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-06T14:41:34,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-06T14:41:34,992 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93, deleteOnExit=true 2024-12-06T14:41:34,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-06T14:41:34,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/test.cache.data in system properties and HBase conf 2024-12-06T14:41:34,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T14:41:34,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/hadoop.log.dir in system properties and HBase conf 2024-12-06T14:41:34,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T14:41:34,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T14:41:34,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T14:41:35,070 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T14:41:35,150 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:41:35,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T14:41:35,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T14:41:35,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T14:41:35,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T14:41:35,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T14:41:35,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T14:41:35,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T14:41:35,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T14:41:35,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T14:41:35,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/nfs.dump.dir in system properties and HBase conf 2024-12-06T14:41:35,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/java.io.tmpdir in system properties and HBase conf 2024-12-06T14:41:35,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T14:41:35,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T14:41:35,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T14:41:36,285 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T14:41:36,352 INFO [Time-limited test {}] log.Log(170): Logging initialized @2459ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T14:41:36,418 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T14:41:36,473 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T14:41:36,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T14:41:36,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T14:41:36,494 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T14:41:36,505 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T14:41:36,507 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/hadoop.log.dir/,AVAILABLE} 2024-12-06T14:41:36,508 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T14:41:36,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/java.io.tmpdir/jetty-localhost-34335-hadoop-hdfs-3_4_1-tests_jar-_-any-7365643529038334901/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T14:41:36,677 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:34335} 2024-12-06T14:41:36,677 INFO [Time-limited test {}] server.Server(415): Started @2785ms 2024-12-06T14:41:37,237 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T14:41:37,243 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T14:41:37,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T14:41:37,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T14:41:37,244 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T14:41:37,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/hadoop.log.dir/,AVAILABLE} 2024-12-06T14:41:37,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T14:41:37,338 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/java.io.tmpdir/jetty-localhost-39049-hadoop-hdfs-3_4_1-tests_jar-_-any-17047838920724562639/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T14:41:37,339 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:39049} 2024-12-06T14:41:37,339 INFO [Time-limited test {}] server.Server(415): Started @3446ms 2024-12-06T14:41:37,384 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T14:41:38,437 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/data/data1/current/BP-1636499811-172.17.0.2-1733496095640/current, will proceed with Du for space computation calculation, 2024-12-06T14:41:38,437 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/data/data2/current/BP-1636499811-172.17.0.2-1733496095640/current, will proceed with Du for space computation calculation, 2024-12-06T14:41:38,469 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T14:41:38,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd053b28cef7d9de with lease ID 0xb29342629174148: Processing first storage report for DS-e985a838-ffbc-4f05-acc2-f100b0b73efa from datanode DatanodeRegistration(127.0.0.1:35987, datanodeUuid=91e1b61b-e56e-40a9-ae1f-956ec261e67b, infoPort=46009, infoSecurePort=0, ipcPort=38897, storageInfo=lv=-57;cid=testClusterID;nsid=467540351;c=1733496095640) 2024-12-06T14:41:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd053b28cef7d9de with lease ID 0xb29342629174148: from storage DS-e985a838-ffbc-4f05-acc2-f100b0b73efa node DatanodeRegistration(127.0.0.1:35987, datanodeUuid=91e1b61b-e56e-40a9-ae1f-956ec261e67b, infoPort=46009, infoSecurePort=0, ipcPort=38897, storageInfo=lv=-57;cid=testClusterID;nsid=467540351;c=1733496095640), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T14:41:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd053b28cef7d9de with lease ID 0xb29342629174148: Processing first storage report for DS-f35dd95f-7213-4f5b-a0cd-1e536840406a from datanode DatanodeRegistration(127.0.0.1:35987, datanodeUuid=91e1b61b-e56e-40a9-ae1f-956ec261e67b, infoPort=46009, infoSecurePort=0, ipcPort=38897, storageInfo=lv=-57;cid=testClusterID;nsid=467540351;c=1733496095640) 2024-12-06T14:41:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd053b28cef7d9de with lease ID 0xb29342629174148: from storage DS-f35dd95f-7213-4f5b-a0cd-1e536840406a node DatanodeRegistration(127.0.0.1:35987, datanodeUuid=91e1b61b-e56e-40a9-ae1f-956ec261e67b, infoPort=46009, infoSecurePort=0, ipcPort=38897, storageInfo=lv=-57;cid=testClusterID;nsid=467540351;c=1733496095640), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T14:41:38,548 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520 2024-12-06T14:41:38,610 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/zookeeper_0, clientPort=63690, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T14:41:38,620 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63690 2024-12-06T14:41:38,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:38,631 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:38,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741825_1001 (size=7) 2024-12-06T14:41:39,270 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 with version=8 2024-12-06T14:41:39,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/hbase-staging 2024-12-06T14:41:39,358 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T14:41:39,651 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:41:39,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:41:39,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:41:39,664 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:41:39,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:41:39,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:41:39,785 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T14:41:39,838 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T14:41:39,846 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T14:41:39,850 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:41:39,874 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5742 (auto-detected) 2024-12-06T14:41:39,876 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T14:41:39,893 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38707 2024-12-06T14:41:39,912 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38707 connecting to ZooKeeper ensemble=127.0.0.1:63690 2024-12-06T14:41:40,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387070x0, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:41:40,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38707-0x101ab174acf0000 connected 2024-12-06T14:41:40,150 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:40,153 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:40,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:41:40,170 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5, hbase.cluster.distributed=false 2024-12-06T14:41:40,190 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:41:40,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38707 2024-12-06T14:41:40,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38707 2024-12-06T14:41:40,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38707 2024-12-06T14:41:40,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38707 2024-12-06T14:41:40,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38707 2024-12-06T14:41:40,284 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:41:40,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:41:40,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:41:40,286 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:41:40,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:41:40,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:41:40,289 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T14:41:40,291 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:41:40,292 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35831 2024-12-06T14:41:40,293 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35831 connecting to ZooKeeper ensemble=127.0.0.1:63690 2024-12-06T14:41:40,294 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:40,297 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:40,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358310x0, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:41:40,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358310x0, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:41:40,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35831-0x101ab174acf0001 connected 2024-12-06T14:41:40,316 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T14:41:40,323 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T14:41:40,325 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T14:41:40,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:41:40,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35831 2024-12-06T14:41:40,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35831 2024-12-06T14:41:40,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35831 2024-12-06T14:41:40,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35831 2024-12-06T14:41:40,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35831 2024-12-06T14:41:40,349 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bb7ea3e46d9c:38707 2024-12-06T14:41:40,350 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:40,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:41:40,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:41:40,430 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:40,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T14:41:40,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:40,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:40,486 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T14:41:40,488 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bb7ea3e46d9c,38707,1733496099493 from backup master directory 2024-12-06T14:41:40,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:40,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:41:40,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:41:40,503 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:41:40,504 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:40,507 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T14:41:40,509 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T14:41:40,556 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/hbase.id] with ID: 346f2f90-4d95-4e78-9b2b-95d06fb591bd 2024-12-06T14:41:40,556 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/.tmp/hbase.id 2024-12-06T14:41:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741826_1002 (size=42) 2024-12-06T14:41:40,971 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/.tmp/hbase.id]:[hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/hbase.id] 2024-12-06T14:41:41,015 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T14:41:41,020 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T14:41:41,036 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-06T14:41:41,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:41,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741827_1003 (size=196) 2024-12-06T14:41:41,525 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:41:41,527 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T14:41:41,541 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:41,544 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:41:41,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741828_1004 (size=1189) 2024-12-06T14:41:41,993 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store 2024-12-06T14:41:42,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741829_1005 (size=34) 2024-12-06T14:41:42,424 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T14:41:42,427 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:41:42,428 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T14:41:42,428 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:41:42,428 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:41:42,429 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T14:41:42,429 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:41:42,429 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:41:42,430 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733496102427Disabling compacts and flushes for region at 1733496102427Disabling writes for close at 1733496102429 (+2 ms)Writing region close event to WAL at 1733496102429Closed at 1733496102429 2024-12-06T14:41:42,432 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/.initializing 2024-12-06T14:41:42,432 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/WALs/bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:42,440 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:41:42,452 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C38707%2C1733496099493, suffix=, logDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/WALs/bb7ea3e46d9c,38707,1733496099493, archiveDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/oldWALs, maxLogs=10 2024-12-06T14:41:42,471 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/WALs/bb7ea3e46d9c,38707,1733496099493/bb7ea3e46d9c%2C38707%2C1733496099493.1733496102456, exclude list is [], retry=0 2024-12-06T14:41:42,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35987,DS-e985a838-ffbc-4f05-acc2-f100b0b73efa,DISK] 2024-12-06T14:41:42,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T14:41:42,523 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/WALs/bb7ea3e46d9c,38707,1733496099493/bb7ea3e46d9c%2C38707%2C1733496099493.1733496102456 2024-12-06T14:41:42,524 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46009:46009)] 2024-12-06T14:41:42,525 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:41:42,525 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:41:42,528 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,529 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T14:41:42,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:42,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:42,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T14:41:42,592 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:42,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:41:42,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T14:41:42,596 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:42,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:41:42,597 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T14:41:42,599 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:42,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:41:42,601 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,604 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,605 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,610 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,611 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,614 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:41:42,619 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:41:42,623 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:41:42,625 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63393418, jitterRate=-0.0553644597530365}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:41:42,630 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733496102540Initializing all the Stores at 1733496102542 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496102542Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496102543 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496102543Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496102543Cleaning up temporary data from old regions at 1733496102611 (+68 ms)Region opened successfully at 1733496102629 (+18 ms) 2024-12-06T14:41:42,631 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T14:41:42,659 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f113b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:41:42,686 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T14:41:42,695 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T14:41:42,695 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T14:41:42,698 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T14:41:42,699 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T14:41:42,703 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-06T14:41:42,703 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T14:41:42,726 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T14:41:42,736 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T14:41:42,806 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T14:41:42,810 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T14:41:42,812 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T14:41:42,820 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T14:41:42,823 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T14:41:42,828 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T14:41:42,841 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T14:41:42,844 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T14:41:42,852 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T14:41:42,873 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T14:41:42,883 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T14:41:42,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:41:42,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:41:42,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:42,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:42,898 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bb7ea3e46d9c,38707,1733496099493, sessionid=0x101ab174acf0000, setting cluster-up flag (Was=false) 2024-12-06T14:41:42,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:42,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:42,968 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T14:41:42,974 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:42,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:42,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:43,031 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T14:41:43,036 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:43,045 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T14:41:43,111 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T14:41:43,120 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T14:41:43,126 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T14:41:43,132 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bb7ea3e46d9c,38707,1733496099493 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T14:41:43,141 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:41:43,141 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:41:43,142 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:41:43,142 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(746): ClusterId : 346f2f90-4d95-4e78-9b2b-95d06fb591bd 2024-12-06T14:41:43,142 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:41:43,142 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bb7ea3e46d9c:0, corePoolSize=10, maxPoolSize=10 2024-12-06T14:41:43,143 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,143 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:41:43,143 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,145 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T14:41:43,146 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733496133146 2024-12-06T14:41:43,148 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T14:41:43,149 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:41:43,149 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T14:41:43,149 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T14:41:43,154 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T14:41:43,154 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T14:41:43,155 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T14:41:43,155 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:43,155 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T14:41:43,155 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T14:41:43,156 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,159 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T14:41:43,159 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T14:41:43,160 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T14:41:43,161 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T14:41:43,161 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T14:41:43,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741831_1007 (size=1321) 2024-12-06T14:41:43,164 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T14:41:43,165 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T14:41:43,166 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496103166,5,FailOnTimeoutGroup] 2024-12-06T14:41:43,167 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496103166,5,FailOnTimeoutGroup] 2024-12-06T14:41:43,167 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,167 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T14:41:43,168 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,169 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T14:41:43,169 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,169 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0728b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:41:43,181 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bb7ea3e46d9c:35831 2024-12-06T14:41:43,185 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T14:41:43,185 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T14:41:43,185 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T14:41:43,188 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,38707,1733496099493 with port=35831, startcode=1733496100253 2024-12-06T14:41:43,197 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T14:41:43,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T14:41:43,236 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38707 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:43,237 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38707 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:43,249 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:41:43,249 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35677 2024-12-06T14:41:43,249 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T14:41:43,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:41:43,263 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] zookeeper.ZKUtil(111): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:43,263 WARN [RS:0;bb7ea3e46d9c:35831 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:41:43,263 INFO [RS:0;bb7ea3e46d9c:35831 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:41:43,263 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:43,266 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,35831,1733496100253] 2024-12-06T14:41:43,293 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T14:41:43,304 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T14:41:43,309 INFO [RS:0;bb7ea3e46d9c:35831 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T14:41:43,309 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,310 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T14:41:43,315 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T14:41:43,316 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,316 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,316 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,316 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,317 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,317 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,317 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:41:43,317 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,317 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,317 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,318 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,318 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,318 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:41:43,318 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:41:43,318 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:41:43,319 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,320 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,320 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,320 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,320 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,320 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,35831,1733496100253-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:41:43,336 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T14:41:43,338 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,35831,1733496100253-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,338 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,338 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.Replication(171): bb7ea3e46d9c,35831,1733496100253 started 2024-12-06T14:41:43,353 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:43,354 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,35831,1733496100253, RpcServer on bb7ea3e46d9c/172.17.0.2:35831, sessionid=0x101ab174acf0001 2024-12-06T14:41:43,354 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T14:41:43,355 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:43,355 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,35831,1733496100253' 2024-12-06T14:41:43,355 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T14:41:43,356 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T14:41:43,357 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T14:41:43,357 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T14:41:43,357 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:43,357 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,35831,1733496100253' 2024-12-06T14:41:43,357 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T14:41:43,358 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T14:41:43,358 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T14:41:43,359 INFO [RS:0;bb7ea3e46d9c:35831 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T14:41:43,359 INFO [RS:0;bb7ea3e46d9c:35831 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T14:41:43,468 INFO [RS:0;bb7ea3e46d9c:35831 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:41:43,472 INFO [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C35831%2C1733496100253, suffix=, logDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253, archiveDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/oldWALs, maxLogs=32 2024-12-06T14:41:43,486 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253/bb7ea3e46d9c%2C35831%2C1733496100253.1733496103474, exclude list is [], retry=0 2024-12-06T14:41:43,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35987,DS-e985a838-ffbc-4f05-acc2-f100b0b73efa,DISK] 2024-12-06T14:41:43,495 INFO [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253/bb7ea3e46d9c%2C35831%2C1733496100253.1733496103474 2024-12-06T14:41:43,496 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46009:46009)] 2024-12-06T14:41:43,566 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T14:41:43,567 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:41:43,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741833_1009 (size=32) 2024-12-06T14:41:43,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:41:43,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T14:41:43,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:41:43,987 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:43,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:43,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T14:41:43,990 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:41:43,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:43,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:43,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T14:41:43,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:41:43,994 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:43,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:43,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T14:41:43,997 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:41:43,997 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:43,998 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:43,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:41:44,000 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740 2024-12-06T14:41:44,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740 2024-12-06T14:41:44,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:41:44,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:41:44,004 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:41:44,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:41:44,010 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:41:44,011 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61041665, jitterRate=-0.0904083102941513}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:41:44,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733496103979Initializing all the Stores at 1733496103982 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496103982Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496103982Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496103982Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496103982Cleaning up temporary data from old regions at 1733496104003 (+21 ms)Region opened successfully at 1733496104013 (+10 ms) 2024-12-06T14:41:44,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:41:44,013 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:41:44,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:41:44,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:41:44,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:41:44,015 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:41:44,015 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496104013Disabling compacts and flushes for region at 1733496104013Disabling writes for close at 1733496104013Writing region close event to WAL at 1733496104014 (+1 ms)Closed at 1733496104015 (+1 ms) 2024-12-06T14:41:44,018 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:41:44,018 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T14:41:44,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T14:41:44,030 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:41:44,033 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T14:41:44,187 DEBUG [bb7ea3e46d9c:38707 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T14:41:44,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:44,204 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,35831,1733496100253, state=OPENING 2024-12-06T14:41:44,259 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T14:41:44,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:44,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:41:44,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:41:44,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:41:44,278 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:41:44,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:41:44,459 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T14:41:44,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T14:41:44,472 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T14:41:44,473 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:41:44,473 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T14:41:44,476 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C35831%2C1733496100253.meta, suffix=.meta, logDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253, archiveDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/oldWALs, maxLogs=32 2024-12-06T14:41:44,490 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253/bb7ea3e46d9c%2C35831%2C1733496100253.meta.1733496104478.meta, exclude list is [], retry=0 2024-12-06T14:41:44,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35987,DS-e985a838-ffbc-4f05-acc2-f100b0b73efa,DISK] 2024-12-06T14:41:44,497 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/WALs/bb7ea3e46d9c,35831,1733496100253/bb7ea3e46d9c%2C35831%2C1733496100253.meta.1733496104478.meta 2024-12-06T14:41:44,497 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46009:46009)] 2024-12-06T14:41:44,497 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:41:44,499 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T14:41:44,501 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T14:41:44,506 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T14:41:44,510 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T14:41:44,511 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:41:44,511 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T14:41:44,511 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T14:41:44,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T14:41:44,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:41:44,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:44,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:44,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T14:41:44,518 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:41:44,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:44,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:44,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T14:41:44,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:41:44,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:44,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:44,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T14:41:44,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:41:44,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:44,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:41:44,524 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:41:44,526 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740 2024-12-06T14:41:44,529 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740 2024-12-06T14:41:44,532 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:41:44,532 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:41:44,533 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:41:44,536 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:41:44,538 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60052466, jitterRate=-0.10514852404594421}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:41:44,538 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T14:41:44,540 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733496104512Writing region info on filesystem at 1733496104512Initializing all the Stores at 1733496104514 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496104514Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496104514Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496104514Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496104514Cleaning up temporary data from old regions at 1733496104532 (+18 ms)Running coprocessor post-open hooks at 1733496104538 (+6 ms)Region opened successfully at 1733496104539 (+1 ms) 2024-12-06T14:41:44,546 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733496104452 2024-12-06T14:41:44,557 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T14:41:44,558 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T14:41:44,559 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:44,562 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,35831,1733496100253, state=OPEN 2024-12-06T14:41:44,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:41:44,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:41:44,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:41:44,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:41:44,646 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:44,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T14:41:44,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,35831,1733496100253 in 364 msec 2024-12-06T14:41:44,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T14:41:44,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 635 msec 2024-12-06T14:41:44,666 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:41:44,666 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T14:41:44,683 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:44,684 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:44,701 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:44,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44705, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:44,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6490 sec 2024-12-06T14:41:44,722 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733496104722, completionTime=-1 2024-12-06T14:41:44,725 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-06T14:41:44,726 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T14:41:44,751 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-06T14:41:44,751 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733496164751 2024-12-06T14:41:44,751 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733496224751 2024-12-06T14:41:44,751 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-06T14:41:44,754 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,38707,1733496099493-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,754 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,38707,1733496099493-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,754 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,38707,1733496099493-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,756 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bb7ea3e46d9c:38707, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,756 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,757 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,762 DEBUG [master/bb7ea3e46d9c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T14:41:44,782 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.278sec 2024-12-06T14:41:44,783 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T14:41:44,785 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T14:41:44,786 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T14:41:44,786 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T14:41:44,787 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T14:41:44,788 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,38707,1733496099493-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:41:44,788 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,38707,1733496099493-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T14:41:44,796 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T14:41:44,797 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T14:41:44,797 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,38707,1733496099493-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:41:44,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:44,854 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T14:41:44,854 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T14:41:44,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:44,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:44,868 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:44,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:44,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:44,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:44,885 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:44,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:44,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:44,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=6343 2024-12-06T14:41:44,951 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:41:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-12-06T14:41:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:41:44,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:41:44,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:41:44,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:41:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T14:41:44,988 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:41:44,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-06T14:41:44,991 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:44,994 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:41:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:41:45,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741835_1011 (size=1061) 2024-12-06T14:41:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:41:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:41:45,411 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:41:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741836_1012 (size=53) 2024-12-06T14:41:45,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:41:45,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 709a255a23a9b4b1363f2ec84dd61843, disabling compactions & flushes 2024-12-06T14:41:45,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. after waiting 0 ms 2024-12-06T14:41:45,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 709a255a23a9b4b1363f2ec84dd61843: Waiting for close lock at 1733496105421Disabling compacts and flushes for region at 1733496105421Disabling writes for close at 1733496105422 (+1 ms)Writing region close event to WAL at 1733496105422Closed at 1733496105422 2024-12-06T14:41:45,424 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:41:45,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733496105424"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496105424"}]},"ts":"1733496105424"} 2024-12-06T14:41:45,435 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:41:45,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:41:45,440 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496105437"}]},"ts":"1733496105437"} 2024-12-06T14:41:45,444 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T14:41:45,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, ASSIGN}] 2024-12-06T14:41:45,449 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, ASSIGN 2024-12-06T14:41:45,451 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=false 2024-12-06T14:41:45,605 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=709a255a23a9b4b1363f2ec84dd61843, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:45,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, ASSIGN because future has completed 2024-12-06T14:41:45,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:41:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:41:45,780 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,780 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:41:45,781 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,781 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:41:45,782 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,782 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,785 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,789 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:41:45,789 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 709a255a23a9b4b1363f2ec84dd61843 columnFamilyName A 2024-12-06T14:41:45,789 DEBUG [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:45,790 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.HStore(327): Store=709a255a23a9b4b1363f2ec84dd61843/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:41:45,791 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,793 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:41:45,793 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 709a255a23a9b4b1363f2ec84dd61843 columnFamilyName B 2024-12-06T14:41:45,794 DEBUG [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:45,795 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.HStore(327): Store=709a255a23a9b4b1363f2ec84dd61843/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:41:45,795 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,797 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:41:45,797 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 709a255a23a9b4b1363f2ec84dd61843 columnFamilyName C 2024-12-06T14:41:45,797 DEBUG [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:41:45,798 INFO [StoreOpener-709a255a23a9b4b1363f2ec84dd61843-1 {}] regionserver.HStore(327): Store=709a255a23a9b4b1363f2ec84dd61843/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:41:45,799 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,799 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,800 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,801 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,803 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,803 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,804 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:41:45,806 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,810 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:41:45,811 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 709a255a23a9b4b1363f2ec84dd61843; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74714433, jitterRate=0.11333180963993073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:41:45,811 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:45,812 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 709a255a23a9b4b1363f2ec84dd61843: Running coprocessor pre-open hook at 1733496105782Writing region info on filesystem at 1733496105782Initializing all the Stores at 1733496105784 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496105784Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496105785 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496105785Cleaning up temporary data from old regions at 1733496105803 (+18 ms)Running coprocessor post-open hooks at 1733496105811 (+8 ms)Region opened successfully at 1733496105812 (+1 ms) 2024-12-06T14:41:45,814 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., pid=6, masterSystemTime=1733496105768 2024-12-06T14:41:45,817 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,817 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:45,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=709a255a23a9b4b1363f2ec84dd61843, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:45,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:41:45,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T14:41:45,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 in 209 msec 2024-12-06T14:41:45,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T14:41:45,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, ASSIGN in 382 msec 2024-12-06T14:41:45,834 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:41:45,835 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496105835"}]},"ts":"1733496105835"} 2024-12-06T14:41:45,838 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T14:41:45,840 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:41:45,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 859 msec 2024-12-06T14:41:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:41:46,141 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:46,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f6a966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc7db18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,155 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,162 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:41:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:41:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f70fa58 2024-12-06T14:41:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:41:46,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:41:46,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,183 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256affd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,208 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,211 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,214 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302b6f41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,220 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89bfc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d3547a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,227 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb5cb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,230 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:41:46,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:41:46,233 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:41:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:41:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:41:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b0f1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:41:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:41:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:41:46,236 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:41:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66945757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:41:46,241 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,241 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,241 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,243 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,244 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,246 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,247 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,247 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,248 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:41:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:41:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:41:46,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-06T14:41:46,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:41:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T14:41:46,258 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,261 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:41:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:46,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:41:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T14:41:46,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:46,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:46,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:46,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:46,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T14:41:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/147a93cbc49f42ba8679af9297dba5f1 is 50, key is test_row_0/A:col10/1733496106338/Put/seqid=0 2024-12-06T14:41:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496166517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496166519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496106478, max timestamp=1733496106520Num uniques 859; 2024-12-06T14:41:46,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:46,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496106478, max timestamp=1733496106527Num uniques 859; 2024-12-06T14:41:46,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T14:41:46,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496106478, max timestamp=1733496106520Num uniques 859; 2024-12-06T14:41:46,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T14:41:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741837_1013 (size=12001) 2024-12-06T14:41:46,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/147a93cbc49f42ba8679af9297dba5f1 2024-12-06T14:41:46,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T14:41:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496166652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496166657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496166671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/052abd5e864245e4956cc6dbe2b46434 is 50, key is test_row_0/B:col10/1733496106338/Put/seqid=0 2024-12-06T14:41:46,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496166690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496166692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741838_1014 (size=12001) 2024-12-06T14:41:46,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/052abd5e864245e4956cc6dbe2b46434 2024-12-06T14:41:46,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496166780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/cce2b86022fc4a42bf27a292dd344fcc is 50, key is test_row_0/C:col10/1733496106338/Put/seqid=0 2024-12-06T14:41:46,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496166780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496166794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741839_1015 (size=12001) 2024-12-06T14:41:46,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/cce2b86022fc4a42bf27a292dd344fcc 2024-12-06T14:41:46,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T14:41:46,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:46,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/147a93cbc49f42ba8679af9297dba5f1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/147a93cbc49f42ba8679af9297dba5f1 2024-12-06T14:41:46,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/147a93cbc49f42ba8679af9297dba5f1, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T14:41:46,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/052abd5e864245e4956cc6dbe2b46434 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/052abd5e864245e4956cc6dbe2b46434 2024-12-06T14:41:46,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/052abd5e864245e4956cc6dbe2b46434, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T14:41:46,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/cce2b86022fc4a42bf27a292dd344fcc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/cce2b86022fc4a42bf27a292dd344fcc 2024-12-06T14:41:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T14:41:46,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/cce2b86022fc4a42bf27a292dd344fcc, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T14:41:46,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for 709a255a23a9b4b1363f2ec84dd61843 in 540ms, sequenceid=12, compaction requested=false 2024-12-06T14:41:46,915 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T14:41:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:46,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-12-06T14:41:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:46,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/321ad75928f74ce8ae3a0a176e4dd0b4 is 50, key is test_row_0/A:col10/1733496106659/Put/seqid=0 2024-12-06T14:41:46,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T14:41:46,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741840_1016 (size=14341) 2024-12-06T14:41:46,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:46,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:46,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:46,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/321ad75928f74ce8ae3a0a176e4dd0b4 2024-12-06T14:41:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496166989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496166992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496166992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496166999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1ab35e9602644d36abc0b8b7af523ec6 is 50, key is test_row_0/B:col10/1733496106659/Put/seqid=0 2024-12-06T14:41:47,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496166999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741841_1017 (size=12001) 2024-12-06T14:41:47,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1ab35e9602644d36abc0b8b7af523ec6 2024-12-06T14:41:47,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/085032950ec641ef9d0dc1b28a319c56 is 50, key is test_row_0/C:col10/1733496106659/Put/seqid=0 2024-12-06T14:41:47,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741842_1018 (size=12001) 2024-12-06T14:41:47,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/085032950ec641ef9d0dc1b28a319c56 2024-12-06T14:41:47,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/321ad75928f74ce8ae3a0a176e4dd0b4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/321ad75928f74ce8ae3a0a176e4dd0b4 2024-12-06T14:41:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496167129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496167130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T14:41:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:47,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:47,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/321ad75928f74ce8ae3a0a176e4dd0b4, entries=200, sequenceid=49, filesize=14.0 K 2024-12-06T14:41:47,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1ab35e9602644d36abc0b8b7af523ec6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1ab35e9602644d36abc0b8b7af523ec6 2024-12-06T14:41:47,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1ab35e9602644d36abc0b8b7af523ec6, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T14:41:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/085032950ec641ef9d0dc1b28a319c56 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/085032950ec641ef9d0dc1b28a319c56 2024-12-06T14:41:47,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/085032950ec641ef9d0dc1b28a319c56, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T14:41:47,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.26 KB/141575, heapSize ~394.18 KB/403640, currentSize=40.25 KB/41220 for 709a255a23a9b4b1363f2ec84dd61843 in 274ms, sequenceid=49, compaction requested=false 2024-12-06T14:41:47,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:47,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T14:41:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:47,305 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-06T14:41:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/3fa8e0bfe4fa4e62ad01c080aa070b20 is 50, key is test_row_0/A:col10/1733496106983/Put/seqid=0 2024-12-06T14:41:47,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:47,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741843_1019 (size=12001) 2024-12-06T14:41:47,376 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/3fa8e0bfe4fa4e62ad01c080aa070b20 2024-12-06T14:41:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T14:41:47,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/fe737ad3b57643778e988f284b5e0253 is 50, key is test_row_0/B:col10/1733496106983/Put/seqid=0 2024-12-06T14:41:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741844_1020 (size=12001) 2024-12-06T14:41:47,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/fe737ad3b57643778e988f284b5e0253 2024-12-06T14:41:47,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496107397, max timestamp=1733496107430Num uniques 859; 2024-12-06T14:41:47,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:41:47,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496107397, max timestamp=1733496107430Num uniques 859; 2024-12-06T14:41:47,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:41:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6440087aa24443f684f375716287a5f0 is 50, key is test_row_0/C:col10/1733496106983/Put/seqid=0 2024-12-06T14:41:47,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496107409, max timestamp=1733496107430Num uniques 859; 2024-12-06T14:41:47,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T14:41:47,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741845_1021 (size=12001) 2024-12-06T14:41:47,500 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6440087aa24443f684f375716287a5f0 2024-12-06T14:41:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/3fa8e0bfe4fa4e62ad01c080aa070b20 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3fa8e0bfe4fa4e62ad01c080aa070b20 2024-12-06T14:41:47,531 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3fa8e0bfe4fa4e62ad01c080aa070b20, entries=150, sequenceid=58, filesize=11.7 K 2024-12-06T14:41:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/fe737ad3b57643778e988f284b5e0253 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/fe737ad3b57643778e988f284b5e0253 2024-12-06T14:41:47,543 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/fe737ad3b57643778e988f284b5e0253, entries=150, sequenceid=58, filesize=11.7 K 2024-12-06T14:41:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6440087aa24443f684f375716287a5f0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6440087aa24443f684f375716287a5f0 2024-12-06T14:41:47,553 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6440087aa24443f684f375716287a5f0, entries=150, sequenceid=58, filesize=11.7 K 2024-12-06T14:41:47,564 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=109.18 KB/111805 for 709a255a23a9b4b1363f2ec84dd61843 in 258ms, sequenceid=58, compaction requested=true 2024-12-06T14:41:47,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:47,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:47,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-06T14:41:47,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-12-06T14:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-06T14:41:47,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:47,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:47,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:47,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:47,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:47,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:47,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T14:41:47,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3050 sec 2024-12-06T14:41:47,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3350 sec 2024-12-06T14:41:47,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/35841d56c27c4b76966d9791fa730cd3 is 50, key is test_row_0/A:col10/1733496107500/Put/seqid=0 2024-12-06T14:41:47,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741846_1022 (size=14341) 2024-12-06T14:41:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496167602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/35841d56c27c4b76966d9791fa730cd3 2024-12-06T14:41:47,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496167608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/bfc10881f7284ad4a29471cba534a962 is 50, key is test_row_0/B:col10/1733496107500/Put/seqid=0 2024-12-06T14:41:47,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496167615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496167617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496167618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741847_1023 (size=12001) 2024-12-06T14:41:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496167730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496167730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496167749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496167749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496167749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496167939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496167949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496167958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496167969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:47,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496167974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/bfc10881f7284ad4a29471cba534a962 2024-12-06T14:41:48,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/56afada2b8364ba493b4f9cf2046f903 is 50, key is test_row_0/C:col10/1733496107500/Put/seqid=0 2024-12-06T14:41:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741848_1024 (size=12001) 2024-12-06T14:41:48,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/56afada2b8364ba493b4f9cf2046f903 2024-12-06T14:41:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/35841d56c27c4b76966d9791fa730cd3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/35841d56c27c4b76966d9791fa730cd3 2024-12-06T14:41:48,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/35841d56c27c4b76966d9791fa730cd3, entries=200, sequenceid=94, filesize=14.0 K 2024-12-06T14:41:48,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/bfc10881f7284ad4a29471cba534a962 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/bfc10881f7284ad4a29471cba534a962 2024-12-06T14:41:48,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/bfc10881f7284ad4a29471cba534a962, entries=150, sequenceid=94, filesize=11.7 K 2024-12-06T14:41:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/56afada2b8364ba493b4f9cf2046f903 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/56afada2b8364ba493b4f9cf2046f903 2024-12-06T14:41:48,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/56afada2b8364ba493b4f9cf2046f903, entries=150, sequenceid=94, filesize=11.7 K 2024-12-06T14:41:48,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=53.67 KB/54960 for 709a255a23a9b4b1363f2ec84dd61843 in 682ms, sequenceid=94, compaction requested=true 2024-12-06T14:41:48,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:48,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:41:48,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:48,272 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:41:48,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:41:48,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:48,272 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:41:48,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:41:48,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:48,279 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:41:48,281 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:41:48,281 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,282 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/147a93cbc49f42ba8679af9297dba5f1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/321ad75928f74ce8ae3a0a176e4dd0b4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3fa8e0bfe4fa4e62ad01c080aa070b20, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/35841d56c27c4b76966d9791fa730cd3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=51.4 K 2024-12-06T14:41:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:48,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:41:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:48,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:41:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:48,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:41:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:48,283 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:48,283 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/052abd5e864245e4956cc6dbe2b46434, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1ab35e9602644d36abc0b8b7af523ec6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/fe737ad3b57643778e988f284b5e0253, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/bfc10881f7284ad4a29471cba534a962] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=46.9 K 2024-12-06T14:41:48,284 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 147a93cbc49f42ba8679af9297dba5f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733496106333 2024-12-06T14:41:48,285 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 052abd5e864245e4956cc6dbe2b46434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733496106333 2024-12-06T14:41:48,286 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321ad75928f74ce8ae3a0a176e4dd0b4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733496106627 2024-12-06T14:41:48,287 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab35e9602644d36abc0b8b7af523ec6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733496106659 2024-12-06T14:41:48,288 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fa8e0bfe4fa4e62ad01c080aa070b20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733496106967 2024-12-06T14:41:48,288 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting fe737ad3b57643778e988f284b5e0253, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733496106967 2024-12-06T14:41:48,289 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35841d56c27c4b76966d9791fa730cd3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496107500 2024-12-06T14:41:48,290 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc10881f7284ad4a29471cba534a962, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496107500 2024-12-06T14:41:48,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/434d22d1e59f49a7b026d4642bf84cf8 is 50, key is test_row_0/A:col10/1733496107600/Put/seqid=0 2024-12-06T14:41:48,342 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:48,343 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/0c57f59affff4671acc7a4a7b900a3ed is 50, key is test_row_0/A:col10/1733496107500/Put/seqid=0 2024-12-06T14:41:48,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741849_1025 (size=14341) 2024-12-06T14:41:48,361 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#14 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:48,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/434d22d1e59f49a7b026d4642bf84cf8 2024-12-06T14:41:48,362 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/d9b6490e9c4e4978881ade98bd148c46 is 50, key is test_row_0/B:col10/1733496107500/Put/seqid=0 2024-12-06T14:41:48,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741850_1026 (size=12139) 2024-12-06T14:41:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T14:41:48,410 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:48,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-06T14:41:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T14:41:48,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:48,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:48,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:48,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496168401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496168402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,426 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/0c57f59affff4671acc7a4a7b900a3ed as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/0c57f59affff4671acc7a4a7b900a3ed 2024-12-06T14:41:48,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496168404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741851_1027 (size=12139) 2024-12-06T14:41:48,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/e52fd5c8fcfe460489c48448ec95c329 is 50, key is test_row_0/B:col10/1733496107600/Put/seqid=0 2024-12-06T14:41:48,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496108312, max timestamp=1733496108411Num uniques 859; 2024-12-06T14:41:48,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:48,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496108312, max timestamp=1733496108411Num uniques 859; 2024-12-06T14:41:48,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:48,466 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/d9b6490e9c4e4978881ade98bd148c46 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/d9b6490e9c4e4978881ade98bd148c46 2024-12-06T14:41:48,476 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 0c57f59affff4671acc7a4a7b900a3ed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:48,476 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:48,479 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=12, startTime=1733496108250; duration=0sec 2024-12-06T14:41:48,480 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:48,480 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:41:48,480 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:41:48,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733496108312, max timestamp=1733496108411Num uniques 859; 2024-12-06T14:41:48,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:48,488 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into d9b6490e9c4e4978881ade98bd148c46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:48,488 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:48,488 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=12, startTime=1733496108269; duration=0sec 2024-12-06T14:41:48,489 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:48,489 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:41:48,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741852_1028 (size=12001) 2024-12-06T14:41:48,491 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:41:48,491 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:41:48,491 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,492 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/cce2b86022fc4a42bf27a292dd344fcc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/085032950ec641ef9d0dc1b28a319c56, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6440087aa24443f684f375716287a5f0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/56afada2b8364ba493b4f9cf2046f903] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=46.9 K 2024-12-06T14:41:48,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/e52fd5c8fcfe460489c48448ec95c329 2024-12-06T14:41:48,492 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce2b86022fc4a42bf27a292dd344fcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733496106333 2024-12-06T14:41:48,494 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085032950ec641ef9d0dc1b28a319c56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733496106659 2024-12-06T14:41:48,495 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6440087aa24443f684f375716287a5f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733496106967 2024-12-06T14:41:48,497 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56afada2b8364ba493b4f9cf2046f903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496107500 2024-12-06T14:41:48,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/048db0a6b4dd4ff693ed0b702ece4b9c is 50, key is test_row_0/C:col10/1733496107600/Put/seqid=0 2024-12-06T14:41:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T14:41:48,550 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#17 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:48,550 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/c0d636cfb18e4231bd2e0d386d134eeb is 50, key is test_row_0/C:col10/1733496107500/Put/seqid=0 2024-12-06T14:41:48,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741854_1030 (size=12139) 2024-12-06T14:41:48,569 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/c0d636cfb18e4231bd2e0d386d134eeb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/c0d636cfb18e4231bd2e0d386d134eeb 2024-12-06T14:41:48,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:48,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741853_1029 (size=12001) 2024-12-06T14:41:48,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496168553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496168551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/048db0a6b4dd4ff693ed0b702ece4b9c 2024-12-06T14:41:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496168556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496168571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,590 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into c0d636cfb18e4231bd2e0d386d134eeb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:48,590 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:48,590 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=12, startTime=1733496108272; duration=0sec 2024-12-06T14:41:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496168574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,590 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:48,590 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/434d22d1e59f49a7b026d4642bf84cf8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/434d22d1e59f49a7b026d4642bf84cf8 2024-12-06T14:41:48,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/434d22d1e59f49a7b026d4642bf84cf8, entries=200, sequenceid=106, filesize=14.0 K 2024-12-06T14:41:48,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/e52fd5c8fcfe460489c48448ec95c329 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/e52fd5c8fcfe460489c48448ec95c329 2024-12-06T14:41:48,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/e52fd5c8fcfe460489c48448ec95c329, entries=150, sequenceid=106, filesize=11.7 K 2024-12-06T14:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/048db0a6b4dd4ff693ed0b702ece4b9c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/048db0a6b4dd4ff693ed0b702ece4b9c 2024-12-06T14:41:48,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/048db0a6b4dd4ff693ed0b702ece4b9c, entries=150, sequenceid=106, filesize=11.7 K 2024-12-06T14:41:48,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 709a255a23a9b4b1363f2ec84dd61843 in 362ms, sequenceid=106, compaction requested=false 2024-12-06T14:41:48,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:48,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-06T14:41:48,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:48,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:48,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:48,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:48,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:48,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:48,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/18ff973c07154165b44e181c0f50a15d is 50, key is test_row_0/A:col10/1733496108552/Put/seqid=0 2024-12-06T14:41:48,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T14:41:48,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496168727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741855_1031 (size=14541) 2024-12-06T14:41:48,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496168729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/18ff973c07154165b44e181c0f50a15d 2024-12-06T14:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496168744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496168746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496168752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1c12d93e9be84cd8a4b05a3f7a4be899 is 50, key is test_row_0/B:col10/1733496108552/Put/seqid=0 2024-12-06T14:41:48,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741856_1032 (size=12151) 2024-12-06T14:41:48,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496168859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496168878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496168879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496168880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:48,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496168880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T14:41:49,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496169069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496169089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496169090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496169098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496169099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1c12d93e9be84cd8a4b05a3f7a4be899 2024-12-06T14:41:49,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/824108c8cef1442989448c1fb507b23b is 50, key is test_row_0/C:col10/1733496108552/Put/seqid=0 2024-12-06T14:41:49,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741857_1033 (size=12151) 2024-12-06T14:41:49,288 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T14:41:49,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496169378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496169399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496169399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496169408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496169418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T14:41:49,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:49,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/824108c8cef1442989448c1fb507b23b 2024-12-06T14:41:49,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/18ff973c07154165b44e181c0f50a15d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/18ff973c07154165b44e181c0f50a15d 2024-12-06T14:41:49,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/18ff973c07154165b44e181c0f50a15d, entries=200, sequenceid=144, filesize=14.2 K 2024-12-06T14:41:49,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1c12d93e9be84cd8a4b05a3f7a4be899 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1c12d93e9be84cd8a4b05a3f7a4be899 2024-12-06T14:41:49,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1c12d93e9be84cd8a4b05a3f7a4be899, entries=150, sequenceid=144, filesize=11.9 K 2024-12-06T14:41:49,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/824108c8cef1442989448c1fb507b23b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/824108c8cef1442989448c1fb507b23b 2024-12-06T14:41:49,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/824108c8cef1442989448c1fb507b23b, entries=150, sequenceid=144, filesize=11.9 K 2024-12-06T14:41:49,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 709a255a23a9b4b1363f2ec84dd61843 in 1068ms, sequenceid=144, compaction requested=true 2024-12-06T14:41:49,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:49,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:41:49,768 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:49,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:49,768 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:49,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:41:49,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:49,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:41:49,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:49,770 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:49,770 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:49,770 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:41:49,770 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:41:49,770 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,770 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,771 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/d9b6490e9c4e4978881ade98bd148c46, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/e52fd5c8fcfe460489c48448ec95c329, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1c12d93e9be84cd8a4b05a3f7a4be899] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=35.4 K 2024-12-06T14:41:49,771 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/0c57f59affff4671acc7a4a7b900a3ed, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/434d22d1e59f49a7b026d4642bf84cf8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/18ff973c07154165b44e181c0f50a15d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=40.1 K 2024-12-06T14:41:49,772 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b6490e9c4e4978881ade98bd148c46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496107500 2024-12-06T14:41:49,772 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c57f59affff4671acc7a4a7b900a3ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496107500 2024-12-06T14:41:49,773 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e52fd5c8fcfe460489c48448ec95c329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733496107600 2024-12-06T14:41:49,773 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434d22d1e59f49a7b026d4642bf84cf8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733496107600 2024-12-06T14:41:49,774 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c12d93e9be84cd8a4b05a3f7a4be899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733496108547 2024-12-06T14:41:49,775 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ff973c07154165b44e181c0f50a15d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733496108547 2024-12-06T14:41:49,799 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:49,800 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/94f4fced348f4844ad9c67019bfaa26c is 50, key is test_row_0/A:col10/1733496108552/Put/seqid=0 2024-12-06T14:41:49,800 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:49,801 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c0d2bf879b3243e3bd1c5122bf3e3776 is 50, key is test_row_0/B:col10/1733496108552/Put/seqid=0 2024-12-06T14:41:49,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741858_1034 (size=12391) 2024-12-06T14:41:49,850 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c0d2bf879b3243e3bd1c5122bf3e3776 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0d2bf879b3243e3bd1c5122bf3e3776 2024-12-06T14:41:49,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T14:41:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,853 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:41:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:49,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:49,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:49,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:49,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741859_1035 (size=12391) 2024-12-06T14:41:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/ac2d513a8b2f4eec8ed0e783ded7bfee is 50, key is test_row_1/A:col10/1733496108741/Put/seqid=0 2024-12-06T14:41:49,877 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/94f4fced348f4844ad9c67019bfaa26c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/94f4fced348f4844ad9c67019bfaa26c 2024-12-06T14:41:49,879 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into c0d2bf879b3243e3bd1c5122bf3e3776(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:49,879 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:49,879 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=13, startTime=1733496109768; duration=0sec 2024-12-06T14:41:49,880 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:49,880 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:41:49,880 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:49,883 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:49,883 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:41:49,883 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:49,884 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/c0d636cfb18e4231bd2e0d386d134eeb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/048db0a6b4dd4ff693ed0b702ece4b9c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/824108c8cef1442989448c1fb507b23b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=35.4 K 2024-12-06T14:41:49,885 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d636cfb18e4231bd2e0d386d134eeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496107500 2024-12-06T14:41:49,886 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 048db0a6b4dd4ff693ed0b702ece4b9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733496107600 2024-12-06T14:41:49,887 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 824108c8cef1442989448c1fb507b23b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733496108547 2024-12-06T14:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:49,893 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 94f4fced348f4844ad9c67019bfaa26c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:49,893 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:49,893 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=13, startTime=1733496109768; duration=0sec 2024-12-06T14:41:49,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741860_1036 (size=9757) 2024-12-06T14:41:49,898 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:49,898 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:41:49,899 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/ac2d513a8b2f4eec8ed0e783ded7bfee 2024-12-06T14:41:49,917 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:49,918 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/a4ebafdfc68542d8afdbd66c5b585867 is 50, key is test_row_0/C:col10/1733496108552/Put/seqid=0 2024-12-06T14:41:49,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/a14b307478f64d6d81e898296a7ed983 is 50, key is test_row_1/B:col10/1733496108741/Put/seqid=0 2024-12-06T14:41:49,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741861_1037 (size=9757) 2024-12-06T14:41:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741862_1038 (size=12391) 2024-12-06T14:41:49,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496109937, max timestamp=1733496109945Num uniques 859; 2024-12-06T14:41:49,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:49,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496109937, max timestamp=1733496109945Num uniques 859; 2024-12-06T14:41:49,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:49,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496109937, max timestamp=1733496109945Num uniques 859; 2024-12-06T14:41:49,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:49,966 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/a4ebafdfc68542d8afdbd66c5b585867 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/a4ebafdfc68542d8afdbd66c5b585867 2024-12-06T14:41:49,986 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into a4ebafdfc68542d8afdbd66c5b585867(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:49,987 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:49,987 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=13, startTime=1733496109769; duration=0sec 2024-12-06T14:41:49,988 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:49,988 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:41:49,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496169988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496169991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496169991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496169997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496169998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,021 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:41:50,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T14:41:50,080 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-06T14:41:50,083 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T14:41:50,083 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-06T14:41:50,084 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T14:41:50,084 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T14:41:50,085 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T14:41:50,085 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-06T14:41:50,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496170110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496170110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496170111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496170111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496170116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496170318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496170318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496170329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496170329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496170329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,347 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/a14b307478f64d6d81e898296a7ed983 2024-12-06T14:41:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/d11cbce456ab4beea137825641d12aa3 is 50, key is test_row_1/C:col10/1733496108741/Put/seqid=0 2024-12-06T14:41:50,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741863_1039 (size=9757) 2024-12-06T14:41:50,385 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/d11cbce456ab4beea137825641d12aa3 2024-12-06T14:41:50,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/ac2d513a8b2f4eec8ed0e783ded7bfee as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/ac2d513a8b2f4eec8ed0e783ded7bfee 2024-12-06T14:41:50,414 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/ac2d513a8b2f4eec8ed0e783ded7bfee, entries=100, sequenceid=156, filesize=9.5 K 2024-12-06T14:41:50,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/a14b307478f64d6d81e898296a7ed983 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/a14b307478f64d6d81e898296a7ed983 2024-12-06T14:41:50,434 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/a14b307478f64d6d81e898296a7ed983, entries=100, sequenceid=156, filesize=9.5 K 2024-12-06T14:41:50,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/d11cbce456ab4beea137825641d12aa3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d11cbce456ab4beea137825641d12aa3 2024-12-06T14:41:50,449 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d11cbce456ab4beea137825641d12aa3, entries=100, sequenceid=156, filesize=9.5 K 2024-12-06T14:41:50,451 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 709a255a23a9b4b1363f2ec84dd61843 in 597ms, sequenceid=156, compaction requested=false 2024-12-06T14:41:50,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:50,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:50,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-06T14:41:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-06T14:41:50,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-06T14:41:50,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0330 sec 2024-12-06T14:41:50,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0420 sec 2024-12-06T14:41:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T14:41:50,568 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:50,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:50,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-06T14:41:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T14:41:50,574 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:50,575 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:50,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:50,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-06T14:41:50,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:50,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/f1ec2500700d4ef093ceca0c311249d6 is 50, key is test_row_0/A:col10/1733496109994/Put/seqid=0 2024-12-06T14:41:50,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741864_1040 (size=14541) 2024-12-06T14:41:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T14:41:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496170691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496170692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496170698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496170700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496170702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:50,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:50,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:50,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:50,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:50,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496170809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496170809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496170809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496170809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496170819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:50,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:50,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T14:41:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496171019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496171019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496171019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496171019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496171028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/f1ec2500700d4ef093ceca0c311249d6 2024-12-06T14:41:51,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/f14b17be82a9443a8430e7535af8bde2 is 50, key is test_row_0/B:col10/1733496109994/Put/seqid=0 2024-12-06T14:41:51,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741865_1041 (size=12151) 2024-12-06T14:41:51,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/f14b17be82a9443a8430e7535af8bde2 2024-12-06T14:41:51,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/0e436ea273234939b3f74f5d12da1a14 is 50, key is test_row_0/C:col10/1733496109994/Put/seqid=0 2024-12-06T14:41:51,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741866_1042 (size=12151) 2024-12-06T14:41:51,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:51,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:51,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T14:41:51,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496171329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496171329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496171329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496171329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496171338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/0e436ea273234939b3f74f5d12da1a14 2024-12-06T14:41:51,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:51,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:51,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:51,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/f1ec2500700d4ef093ceca0c311249d6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/f1ec2500700d4ef093ceca0c311249d6 2024-12-06T14:41:51,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/f1ec2500700d4ef093ceca0c311249d6, entries=200, sequenceid=194, filesize=14.2 K 2024-12-06T14:41:51,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/f14b17be82a9443a8430e7535af8bde2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f14b17be82a9443a8430e7535af8bde2 2024-12-06T14:41:51,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f14b17be82a9443a8430e7535af8bde2, entries=150, sequenceid=194, filesize=11.9 K 2024-12-06T14:41:51,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/0e436ea273234939b3f74f5d12da1a14 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/0e436ea273234939b3f74f5d12da1a14 2024-12-06T14:41:51,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/0e436ea273234939b3f74f5d12da1a14, entries=150, sequenceid=194, filesize=11.9 K 2024-12-06T14:41:51,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 709a255a23a9b4b1363f2ec84dd61843 in 934ms, sequenceid=194, compaction requested=true 2024-12-06T14:41:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:41:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:51,569 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:41:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:51,570 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:41:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:51,572 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:51,572 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:51,572 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:41:51,572 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:41:51,572 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,572 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,573 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0d2bf879b3243e3bd1c5122bf3e3776, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/a14b307478f64d6d81e898296a7ed983, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f14b17be82a9443a8430e7535af8bde2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=33.5 K 2024-12-06T14:41:51,573 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/94f4fced348f4844ad9c67019bfaa26c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/ac2d513a8b2f4eec8ed0e783ded7bfee, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/f1ec2500700d4ef093ceca0c311249d6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=35.8 K 2024-12-06T14:41:51,573 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d2bf879b3243e3bd1c5122bf3e3776, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733496108547 2024-12-06T14:41:51,574 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94f4fced348f4844ad9c67019bfaa26c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733496108547 2024-12-06T14:41:51,574 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a14b307478f64d6d81e898296a7ed983, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733496108741 2024-12-06T14:41:51,574 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac2d513a8b2f4eec8ed0e783ded7bfee, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733496108741 2024-12-06T14:41:51,575 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f14b17be82a9443a8430e7535af8bde2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496109982 2024-12-06T14:41:51,576 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ec2500700d4ef093ceca0c311249d6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496109982 2024-12-06T14:41:51,597 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:51,598 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/90d3501e4f284704893450d112f28c6c is 50, key is test_row_0/B:col10/1733496109994/Put/seqid=0 2024-12-06T14:41:51,610 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:51,611 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/50fadea7a9ed43f5b0663662decd4105 is 50, key is test_row_0/A:col10/1733496109994/Put/seqid=0 2024-12-06T14:41:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741868_1044 (size=12493) 2024-12-06T14:41:51,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741867_1043 (size=12493) 2024-12-06T14:41:51,651 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/90d3501e4f284704893450d112f28c6c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/90d3501e4f284704893450d112f28c6c 2024-12-06T14:41:51,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T14:41:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,666 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:41:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:51,667 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into 90d3501e4f284704893450d112f28c6c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:51,667 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:51,667 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=13, startTime=1733496111569; duration=0sec 2024-12-06T14:41:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:51,668 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:51,668 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:41:51,668 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:51,671 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:51,671 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:41:51,671 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:51,671 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/a4ebafdfc68542d8afdbd66c5b585867, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d11cbce456ab4beea137825641d12aa3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/0e436ea273234939b3f74f5d12da1a14] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=33.5 K 2024-12-06T14:41:51,674 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ebafdfc68542d8afdbd66c5b585867, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733496108547 2024-12-06T14:41:51,676 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d11cbce456ab4beea137825641d12aa3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733496108741 2024-12-06T14:41:51,677 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e436ea273234939b3f74f5d12da1a14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496109982 2024-12-06T14:41:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/915788bc52e54f1c9f980f4dec83fc08 is 50, key is test_row_0/A:col10/1733496110662/Put/seqid=0 2024-12-06T14:41:51,699 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:51,699 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/8e7051b48aaf47218f13986e4bc5219b is 50, key is test_row_0/C:col10/1733496109994/Put/seqid=0 2024-12-06T14:41:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T14:41:51,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741869_1045 (size=12151) 2024-12-06T14:41:51,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741870_1046 (size=12493) 2024-12-06T14:41:51,735 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/8e7051b48aaf47218f13986e4bc5219b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/8e7051b48aaf47218f13986e4bc5219b 2024-12-06T14:41:51,747 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into 8e7051b48aaf47218f13986e4bc5219b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:51,747 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:51,747 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=13, startTime=1733496111570; duration=0sec 2024-12-06T14:41:51,747 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:51,747 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:41:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:51,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733496111857, max timestamp=1733496111868Num uniques 859; 2024-12-06T14:41:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733496111857, max timestamp=1733496111868Num uniques 859; 2024-12-06T14:41:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T14:41:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T14:41:51,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733496111857, max timestamp=1733496111868Num uniques 859; 2024-12-06T14:41:51,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T14:41:51,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496171917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496171917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496171919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496171923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496171925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496172028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496172028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496172029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,039 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/50fadea7a9ed43f5b0663662decd4105 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/50fadea7a9ed43f5b0663662decd4105 2024-12-06T14:41:52,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496172038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496172038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,050 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 50fadea7a9ed43f5b0663662decd4105(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:52,050 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:52,050 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=13, startTime=1733496111569; duration=0sec 2024-12-06T14:41:52,051 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:52,051 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:41:52,117 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/915788bc52e54f1c9f980f4dec83fc08 2024-12-06T14:41:52,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/be392f6fadba48219abe4718c29c4cf2 is 50, key is test_row_0/B:col10/1733496110662/Put/seqid=0 2024-12-06T14:41:52,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741871_1047 (size=12151) 2024-12-06T14:41:52,135 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/be392f6fadba48219abe4718c29c4cf2 2024-12-06T14:41:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1 is 50, key is test_row_0/C:col10/1733496110662/Put/seqid=0 2024-12-06T14:41:52,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741872_1048 (size=12151) 2024-12-06T14:41:52,189 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1 2024-12-06T14:41:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/915788bc52e54f1c9f980f4dec83fc08 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/915788bc52e54f1c9f980f4dec83fc08 2024-12-06T14:41:52,210 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/915788bc52e54f1c9f980f4dec83fc08, entries=150, sequenceid=206, filesize=11.9 K 2024-12-06T14:41:52,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/be392f6fadba48219abe4718c29c4cf2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/be392f6fadba48219abe4718c29c4cf2 2024-12-06T14:41:52,219 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/be392f6fadba48219abe4718c29c4cf2, entries=150, sequenceid=206, filesize=11.9 K 2024-12-06T14:41:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1 2024-12-06T14:41:52,235 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1, entries=150, sequenceid=206, filesize=11.9 K 2024-12-06T14:41:52,237 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 709a255a23a9b4b1363f2ec84dd61843 in 571ms, sequenceid=206, compaction requested=false 2024-12-06T14:41:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-06T14:41:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-06T14:41:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:52,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-06T14:41:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:52,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-06T14:41:52,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6670 sec 2024-12-06T14:41:52,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6800 sec 2024-12-06T14:41:52,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496172255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/6719bb0589074bb28678eda6b440a09c is 50, key is test_row_0/A:col10/1733496111878/Put/seqid=0 2024-12-06T14:41:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496172257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496172257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496172261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496172262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741873_1049 (size=16931) 2024-12-06T14:41:52,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496172368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496172378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496172379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496172379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496172379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496172578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496172588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496172588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496172588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496172588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/6719bb0589074bb28678eda6b440a09c 2024-12-06T14:41:52,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/7108fa419dbf4d9e9b53ccdcbfc3316c is 50, key is test_row_0/B:col10/1733496111878/Put/seqid=0 2024-12-06T14:41:52,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741874_1050 (size=12151) 2024-12-06T14:41:52,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/7108fa419dbf4d9e9b53ccdcbfc3316c 2024-12-06T14:41:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T14:41:52,719 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:52,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-06T14:41:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T14:41:52,723 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:52,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:52,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:52,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/ffea5d0ffc9d41aca7b4b66e4c856e30 is 50, key is test_row_0/C:col10/1733496111878/Put/seqid=0 2024-12-06T14:41:52,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741875_1051 (size=12151) 2024-12-06T14:41:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T14:41:52,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T14:41:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:52,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:52,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496172888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496172898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496172899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496172899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496172899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T14:41:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T14:41:53,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/ffea5d0ffc9d41aca7b4b66e4c856e30 2024-12-06T14:41:53,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/6719bb0589074bb28678eda6b440a09c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6719bb0589074bb28678eda6b440a09c 2024-12-06T14:41:53,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6719bb0589074bb28678eda6b440a09c, entries=250, sequenceid=244, filesize=16.5 K 2024-12-06T14:41:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/7108fa419dbf4d9e9b53ccdcbfc3316c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/7108fa419dbf4d9e9b53ccdcbfc3316c 2024-12-06T14:41:53,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T14:41:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:53,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/7108fa419dbf4d9e9b53ccdcbfc3316c, entries=150, sequenceid=244, filesize=11.9 K 2024-12-06T14:41:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/ffea5d0ffc9d41aca7b4b66e4c856e30 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ffea5d0ffc9d41aca7b4b66e4c856e30 2024-12-06T14:41:53,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ffea5d0ffc9d41aca7b4b66e4c856e30, entries=150, sequenceid=244, filesize=11.9 K 2024-12-06T14:41:53,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 709a255a23a9b4b1363f2ec84dd61843 in 961ms, sequenceid=244, compaction requested=true 2024-12-06T14:41:53,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:41:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:53,204 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:41:53,204 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:41:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:53,205 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:53,205 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:53,205 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:41:53,205 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,205 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:41:53,205 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/90d3501e4f284704893450d112f28c6c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/be392f6fadba48219abe4718c29c4cf2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/7108fa419dbf4d9e9b53ccdcbfc3316c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=35.9 K 2024-12-06T14:41:53,205 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,206 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/50fadea7a9ed43f5b0663662decd4105, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/915788bc52e54f1c9f980f4dec83fc08, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6719bb0589074bb28678eda6b440a09c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=40.6 K 2024-12-06T14:41:53,207 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50fadea7a9ed43f5b0663662decd4105, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496109982 2024-12-06T14:41:53,208 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d3501e4f284704893450d112f28c6c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496109982 2024-12-06T14:41:53,208 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915788bc52e54f1c9f980f4dec83fc08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733496110662 2024-12-06T14:41:53,209 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting be392f6fadba48219abe4718c29c4cf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733496110662 2024-12-06T14:41:53,209 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6719bb0589074bb28678eda6b440a09c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733496111878 2024-12-06T14:41:53,209 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 7108fa419dbf4d9e9b53ccdcbfc3316c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733496111878 2024-12-06T14:41:53,226 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:53,227 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/f04d11e0d32147c5971a1d740a5cfae5 is 50, key is test_row_0/B:col10/1733496111878/Put/seqid=0 2024-12-06T14:41:53,233 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:53,233 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/6bc9ac7034f74e86a65132b8cad93091 is 50, key is test_row_0/A:col10/1733496111878/Put/seqid=0 2024-12-06T14:41:53,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741876_1052 (size=12595) 2024-12-06T14:41:53,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741877_1053 (size=12595) 2024-12-06T14:41:53,268 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/f04d11e0d32147c5971a1d740a5cfae5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f04d11e0d32147c5971a1d740a5cfae5 2024-12-06T14:41:53,280 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/6bc9ac7034f74e86a65132b8cad93091 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6bc9ac7034f74e86a65132b8cad93091 2024-12-06T14:41:53,280 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into f04d11e0d32147c5971a1d740a5cfae5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:53,280 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:53,280 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=13, startTime=1733496113204; duration=0sec 2024-12-06T14:41:53,280 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:53,280 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:41:53,281 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:53,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:53,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:41:53,283 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,283 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/8e7051b48aaf47218f13986e4bc5219b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ffea5d0ffc9d41aca7b4b66e4c856e30] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=35.9 K 2024-12-06T14:41:53,287 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7051b48aaf47218f13986e4bc5219b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496109982 2024-12-06T14:41:53,287 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e05b0a0b05de4e0e8e6ffbe08a27d0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733496110662 2024-12-06T14:41:53,288 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ffea5d0ffc9d41aca7b4b66e4c856e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733496111878 2024-12-06T14:41:53,294 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 6bc9ac7034f74e86a65132b8cad93091(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:53,294 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:53,294 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=13, startTime=1733496113203; duration=0sec 2024-12-06T14:41:53,295 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:53,295 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:41:53,306 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:53,306 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/d45cc7df40454703a3cb30d429cc0556 is 50, key is test_row_0/C:col10/1733496111878/Put/seqid=0 2024-12-06T14:41:53,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T14:41:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:53,346 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:41:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:53,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741878_1054 (size=12595) 2024-12-06T14:41:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T14:41:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/760d0fc260c442b185fbba842d644c91 is 50, key is test_row_0/A:col10/1733496112257/Put/seqid=0 2024-12-06T14:41:53,364 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/d45cc7df40454703a3cb30d429cc0556 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d45cc7df40454703a3cb30d429cc0556 2024-12-06T14:41:53,377 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into d45cc7df40454703a3cb30d429cc0556(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:53,377 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:53,377 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=13, startTime=1733496113204; duration=0sec 2024-12-06T14:41:53,377 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:53,377 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:41:53,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741879_1055 (size=12151) 2024-12-06T14:41:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:53,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733496113411, max timestamp=1733496113436Num uniques 859; 2024-12-06T14:41:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-06T14:41:53,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733496113411, max timestamp=1733496113436Num uniques 859; 2024-12-06T14:41:53,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-06T14:41:53,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496113430, max timestamp=1733496113436Num uniques 859; 2024-12-06T14:41:53,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:41:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496173459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496173459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496173460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496173465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496173466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496173568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496173579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496173579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496173579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496173579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496173778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,789 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/760d0fc260c442b185fbba842d644c91 2024-12-06T14:41:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496173788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496173788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496173789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496173789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/43c886282b294947b726ce949a295021 is 50, key is test_row_0/B:col10/1733496112257/Put/seqid=0 2024-12-06T14:41:53,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741880_1056 (size=12151) 2024-12-06T14:41:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T14:41:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496174088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496174099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496174099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496174099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496174099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,206 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/43c886282b294947b726ce949a295021 2024-12-06T14:41:54,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f4bddf081eae4f809ad4be7611bdc252 is 50, key is test_row_0/C:col10/1733496112257/Put/seqid=0 2024-12-06T14:41:54,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741881_1057 (size=12151) 2024-12-06T14:41:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496174598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496174609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496174609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496174609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496174618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:54,624 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f4bddf081eae4f809ad4be7611bdc252 2024-12-06T14:41:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/760d0fc260c442b185fbba842d644c91 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/760d0fc260c442b185fbba842d644c91 2024-12-06T14:41:54,643 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/760d0fc260c442b185fbba842d644c91, entries=150, sequenceid=256, filesize=11.9 K 2024-12-06T14:41:54,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/43c886282b294947b726ce949a295021 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/43c886282b294947b726ce949a295021 2024-12-06T14:41:54,650 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/43c886282b294947b726ce949a295021, entries=150, sequenceid=256, filesize=11.9 K 2024-12-06T14:41:54,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f4bddf081eae4f809ad4be7611bdc252 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f4bddf081eae4f809ad4be7611bdc252 2024-12-06T14:41:54,660 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f4bddf081eae4f809ad4be7611bdc252, entries=150, sequenceid=256, filesize=11.9 K 2024-12-06T14:41:54,661 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 709a255a23a9b4b1363f2ec84dd61843 in 1316ms, sequenceid=256, compaction requested=false 2024-12-06T14:41:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-06T14:41:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-06T14:41:54,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-06T14:41:54,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9400 sec 2024-12-06T14:41:54,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.9480 sec 2024-12-06T14:41:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T14:41:54,869 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:54,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-06T14:41:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T14:41:54,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:54,873 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:54,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T14:41:55,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T14:41:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:55,027 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-06T14:41:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:55,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:55,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/8d188a90570b4810b4e8f8ca58cf074f is 50, key is test_row_0/A:col10/1733496113463/Put/seqid=0 2024-12-06T14:41:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741882_1058 (size=12301) 2024-12-06T14:41:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T14:41:55,439 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/8d188a90570b4810b4e8f8ca58cf074f 2024-12-06T14:41:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/ce89ade3144c4c859a23ea411036eb21 is 50, key is test_row_0/B:col10/1733496113463/Put/seqid=0 2024-12-06T14:41:55,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741883_1059 (size=12301) 2024-12-06T14:41:55,467 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/ce89ade3144c4c859a23ea411036eb21 2024-12-06T14:41:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/e7a66e41f4474c508aed1d9236e6de3b is 50, key is test_row_0/C:col10/1733496113463/Put/seqid=0 2024-12-06T14:41:55,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741884_1060 (size=12301) 2024-12-06T14:41:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T14:41:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496175628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496175628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496175630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,635 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:41:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496175631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496175632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,635 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496175748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496175749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496175749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:55,890 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/e7a66e41f4474c508aed1d9236e6de3b 2024-12-06T14:41:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/8d188a90570b4810b4e8f8ca58cf074f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8d188a90570b4810b4e8f8ca58cf074f 2024-12-06T14:41:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8d188a90570b4810b4e8f8ca58cf074f, entries=150, sequenceid=292, filesize=12.0 K 2024-12-06T14:41:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/ce89ade3144c4c859a23ea411036eb21 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/ce89ade3144c4c859a23ea411036eb21 2024-12-06T14:41:55,911 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/ce89ade3144c4c859a23ea411036eb21, entries=150, sequenceid=292, filesize=12.0 K 2024-12-06T14:41:55,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/e7a66e41f4474c508aed1d9236e6de3b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e7a66e41f4474c508aed1d9236e6de3b 2024-12-06T14:41:55,924 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e7a66e41f4474c508aed1d9236e6de3b, entries=150, sequenceid=292, filesize=12.0 K 2024-12-06T14:41:55,926 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 709a255a23a9b4b1363f2ec84dd61843 in 899ms, sequenceid=292, compaction requested=true 2024-12-06T14:41:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-06T14:41:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-06T14:41:55,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-06T14:41:55,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0560 sec 2024-12-06T14:41:55,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0640 sec 2024-12-06T14:41:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:55,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:41:55,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:55,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:55,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:55,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:55,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/33a21a42f8b94c3db7f65f267a9df2bf is 50, key is test_row_0/A:col10/1733496115961/Put/seqid=0 2024-12-06T14:41:55,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741885_1061 (size=17181) 2024-12-06T14:41:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T14:41:56,009 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:56,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-06T14:41:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T14:41:56,015 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:56,016 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:56,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:56,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733496115998, max timestamp=1733496116012Num uniques 859; 2024-12-06T14:41:56,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:41:56,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496115998, max timestamp=1733496116012Num uniques 859; 2024-12-06T14:41:56,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496115998, max timestamp=1733496116012Num uniques 859; 2024-12-06T14:41:56,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:41:56,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:41:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496176045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496176047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496176047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T14:41:56,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496176159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496176159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496176159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:56,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T14:41:56,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496176369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496176369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496176369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/33a21a42f8b94c3db7f65f267a9df2bf 2024-12-06T14:41:56,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0a84a99425b04f4c95ebfb456f22e23d is 50, key is test_row_0/B:col10/1733496115961/Put/seqid=0 2024-12-06T14:41:56,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741886_1062 (size=12301) 2024-12-06T14:41:56,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T14:41:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496176678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496176679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496176679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:56,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0a84a99425b04f4c95ebfb456f22e23d 2024-12-06T14:41:56,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/7ed180ae92df4a60be2353d5050e8aa5 is 50, key is test_row_0/C:col10/1733496115961/Put/seqid=0 2024-12-06T14:41:56,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741887_1063 (size=12301) 2024-12-06T14:41:56,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:56,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T14:41:57,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496177188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496177188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496177188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/7ed180ae92df4a60be2353d5050e8aa5 2024-12-06T14:41:57,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/33a21a42f8b94c3db7f65f267a9df2bf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/33a21a42f8b94c3db7f65f267a9df2bf 2024-12-06T14:41:57,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/33a21a42f8b94c3db7f65f267a9df2bf, entries=250, sequenceid=304, filesize=16.8 K 2024-12-06T14:41:57,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0a84a99425b04f4c95ebfb456f22e23d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0a84a99425b04f4c95ebfb456f22e23d 2024-12-06T14:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:57,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0a84a99425b04f4c95ebfb456f22e23d, entries=150, sequenceid=304, filesize=12.0 K 2024-12-06T14:41:57,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/7ed180ae92df4a60be2353d5050e8aa5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7ed180ae92df4a60be2353d5050e8aa5 2024-12-06T14:41:57,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7ed180ae92df4a60be2353d5050e8aa5, entries=150, sequenceid=304, filesize=12.0 K 2024-12-06T14:41:57,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 709a255a23a9b4b1363f2ec84dd61843 in 1305ms, sequenceid=304, compaction requested=true 2024-12-06T14:41:57,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:57,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:41:57,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:57,267 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:41:57,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:41:57,267 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:41:57,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:57,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:41:57,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:57,269 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54228 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:41:57,269 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:41:57,269 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:41:57,269 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:41:57,269 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,269 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,269 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f04d11e0d32147c5971a1d740a5cfae5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/43c886282b294947b726ce949a295021, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/ce89ade3144c4c859a23ea411036eb21, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0a84a99425b04f4c95ebfb456f22e23d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=48.2 K 2024-12-06T14:41:57,269 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6bc9ac7034f74e86a65132b8cad93091, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/760d0fc260c442b185fbba842d644c91, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8d188a90570b4810b4e8f8ca58cf074f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/33a21a42f8b94c3db7f65f267a9df2bf] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=53.0 K 2024-12-06T14:41:57,270 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f04d11e0d32147c5971a1d740a5cfae5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733496111878 2024-12-06T14:41:57,270 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc9ac7034f74e86a65132b8cad93091, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733496111878 2024-12-06T14:41:57,270 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c886282b294947b726ce949a295021, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733496112252 2024-12-06T14:41:57,270 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760d0fc260c442b185fbba842d644c91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733496112252 2024-12-06T14:41:57,270 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d188a90570b4810b4e8f8ca58cf074f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733496113456 2024-12-06T14:41:57,270 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ce89ade3144c4c859a23ea411036eb21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733496113456 2024-12-06T14:41:57,271 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a84a99425b04f4c95ebfb456f22e23d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733496115624 2024-12-06T14:41:57,272 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33a21a42f8b94c3db7f65f267a9df2bf, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733496115624 2024-12-06T14:41:57,285 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:57,286 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/36599200808d487dba769548980d6273 is 50, key is test_row_0/B:col10/1733496115961/Put/seqid=0 2024-12-06T14:41:57,286 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#52 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:57,287 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/e24d6567057047beb98a59de00739cdd is 50, key is test_row_0/A:col10/1733496115961/Put/seqid=0 2024-12-06T14:41:57,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741889_1065 (size=12881) 2024-12-06T14:41:57,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741888_1064 (size=12881) 2024-12-06T14:41:57,306 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/e24d6567057047beb98a59de00739cdd as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/e24d6567057047beb98a59de00739cdd 2024-12-06T14:41:57,315 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/36599200808d487dba769548980d6273 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/36599200808d487dba769548980d6273 2024-12-06T14:41:57,319 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into e24d6567057047beb98a59de00739cdd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:57,319 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:57,319 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=12, startTime=1733496117267; duration=0sec 2024-12-06T14:41:57,320 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:57,320 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:41:57,320 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:41:57,321 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:41:57,322 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:41:57,322 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,322 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d45cc7df40454703a3cb30d429cc0556, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f4bddf081eae4f809ad4be7611bdc252, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e7a66e41f4474c508aed1d9236e6de3b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7ed180ae92df4a60be2353d5050e8aa5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=48.2 K 2024-12-06T14:41:57,323 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d45cc7df40454703a3cb30d429cc0556, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733496111878 2024-12-06T14:41:57,324 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bddf081eae4f809ad4be7611bdc252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733496112252 2024-12-06T14:41:57,324 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a66e41f4474c508aed1d9236e6de3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733496113456 2024-12-06T14:41:57,324 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into 36599200808d487dba769548980d6273(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:57,324 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:57,324 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=12, startTime=1733496117267; duration=0sec 2024-12-06T14:41:57,324 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:57,324 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed180ae92df4a60be2353d5050e8aa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733496115624 2024-12-06T14:41:57,324 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:41:57,335 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:57,336 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/acb44ab48c204dbba76c86adc7cefda2 is 50, key is test_row_0/C:col10/1733496115961/Put/seqid=0 2024-12-06T14:41:57,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741890_1066 (size=12881) 2024-12-06T14:41:57,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T14:41:57,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,402 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-12-06T14:41:57,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:41:57,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/c0eade68238a40f2b02b3043f1c8b8d9 is 50, key is test_row_0/A:col10/1733496116045/Put/seqid=0 2024-12-06T14:41:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741891_1067 (size=12301) 2024-12-06T14:41:57,414 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/c0eade68238a40f2b02b3043f1c8b8d9 2024-12-06T14:41:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/768af12189da4915b3536bdfa2fdbd6e is 50, key is test_row_0/B:col10/1733496116045/Put/seqid=0 2024-12-06T14:41:57,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741892_1068 (size=12301) 2024-12-06T14:41:57,429 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/768af12189da4915b3536bdfa2fdbd6e 2024-12-06T14:41:57,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f299ad024d274fc78c58eb789a3a8569 is 50, key is test_row_0/C:col10/1733496116045/Put/seqid=0 2024-12-06T14:41:57,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741893_1069 (size=12301) 2024-12-06T14:41:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496177673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496177674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,750 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/acb44ab48c204dbba76c86adc7cefda2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/acb44ab48c204dbba76c86adc7cefda2 2024-12-06T14:41:57,757 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into acb44ab48c204dbba76c86adc7cefda2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:57,757 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:57,757 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=12, startTime=1733496117267; duration=0sec 2024-12-06T14:41:57,757 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:57,757 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:41:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496177789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496177789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:57,849 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f299ad024d274fc78c58eb789a3a8569 2024-12-06T14:41:57,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/c0eade68238a40f2b02b3043f1c8b8d9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c0eade68238a40f2b02b3043f1c8b8d9 2024-12-06T14:41:57,863 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c0eade68238a40f2b02b3043f1c8b8d9, entries=150, sequenceid=339, filesize=12.0 K 2024-12-06T14:41:57,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/768af12189da4915b3536bdfa2fdbd6e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/768af12189da4915b3536bdfa2fdbd6e 2024-12-06T14:41:57,870 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/768af12189da4915b3536bdfa2fdbd6e, entries=150, sequenceid=339, filesize=12.0 K 2024-12-06T14:41:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f299ad024d274fc78c58eb789a3a8569 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f299ad024d274fc78c58eb789a3a8569 2024-12-06T14:41:57,877 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f299ad024d274fc78c58eb789a3a8569, entries=150, sequenceid=339, filesize=12.0 K 2024-12-06T14:41:57,878 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 709a255a23a9b4b1363f2ec84dd61843 in 476ms, sequenceid=339, compaction requested=false 2024-12-06T14:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-06T14:41:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-06T14:41:57,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-06T14:41:57,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-12-06T14:41:57,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8750 sec 2024-12-06T14:41:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:58,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T14:41:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:58,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/100d24687f5c44348de6ef1792d7314d is 50, key is test_row_0/A:col10/1733496118000/Put/seqid=0 2024-12-06T14:41:58,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741894_1070 (size=14741) 2024-12-06T14:41:58,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/100d24687f5c44348de6ef1792d7314d 2024-12-06T14:41:58,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496178047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496178050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/93a0309900d44d9e9205cc2100cd3fa8 is 50, key is test_row_0/B:col10/1733496118000/Put/seqid=0 2024-12-06T14:41:58,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741895_1071 (size=12301) 2024-12-06T14:41:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T14:41:58,159 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:41:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496178158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496178159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-06T14:41:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T14:41:58,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:41:58,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:41:58,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:41:58,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496178198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,202 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:41:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496178208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496178208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,212 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:41:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,213 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:41:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T14:41:58,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T14:41:58,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:58,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496178368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496178368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/93a0309900d44d9e9205cc2100cd3fa8 2024-12-06T14:41:58,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T14:41:58,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T14:41:58,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/840c036da67c4cf1aaf57c965946324b is 50, key is test_row_0/C:col10/1733496118000/Put/seqid=0 2024-12-06T14:41:58,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741896_1072 (size=12301) 2024-12-06T14:41:58,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T14:41:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496178678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496178679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:58,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T14:41:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:41:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T14:41:58,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/840c036da67c4cf1aaf57c965946324b 2024-12-06T14:41:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/100d24687f5c44348de6ef1792d7314d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/100d24687f5c44348de6ef1792d7314d 2024-12-06T14:41:58,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/100d24687f5c44348de6ef1792d7314d, entries=200, sequenceid=354, filesize=14.4 K 2024-12-06T14:41:58,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/93a0309900d44d9e9205cc2100cd3fa8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/93a0309900d44d9e9205cc2100cd3fa8 2024-12-06T14:41:58,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/93a0309900d44d9e9205cc2100cd3fa8, entries=150, sequenceid=354, filesize=12.0 K 2024-12-06T14:41:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/840c036da67c4cf1aaf57c965946324b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/840c036da67c4cf1aaf57c965946324b 2024-12-06T14:41:58,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/840c036da67c4cf1aaf57c965946324b, entries=150, sequenceid=354, filesize=12.0 K 2024-12-06T14:41:58,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 709a255a23a9b4b1363f2ec84dd61843 in 921ms, sequenceid=354, compaction requested=true 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:58,923 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:41:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:58,924 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:58,925 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:58,925 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:41:58,925 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,925 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/36599200808d487dba769548980d6273, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/768af12189da4915b3536bdfa2fdbd6e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/93a0309900d44d9e9205cc2100cd3fa8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=36.6 K 2024-12-06T14:41:58,925 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 36599200808d487dba769548980d6273, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733496115624 2024-12-06T14:41:58,926 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:58,926 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 768af12189da4915b3536bdfa2fdbd6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496116038 2024-12-06T14:41:58,926 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:41:58,926 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,926 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/e24d6567057047beb98a59de00739cdd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c0eade68238a40f2b02b3043f1c8b8d9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/100d24687f5c44348de6ef1792d7314d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=39.0 K 2024-12-06T14:41:58,926 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a0309900d44d9e9205cc2100cd3fa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733496117671 2024-12-06T14:41:58,926 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting e24d6567057047beb98a59de00739cdd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733496115624 2024-12-06T14:41:58,927 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0eade68238a40f2b02b3043f1c8b8d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496116038 2024-12-06T14:41:58,927 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100d24687f5c44348de6ef1792d7314d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733496117671 2024-12-06T14:41:58,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T14:41:58,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,932 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:41:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:58,936 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:58,936 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/b49ab87d88eb4987a31b05091d01dd3c is 50, key is test_row_0/B:col10/1733496118000/Put/seqid=0 2024-12-06T14:41:58,939 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:58,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/8aa74fc18b464a84bb07f2b8d4b70c3a is 50, key is test_row_0/A:col10/1733496118030/Put/seqid=0 2024-12-06T14:41:58,940 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/52697209e92240078274585eda571287 is 50, key is test_row_0/A:col10/1733496118000/Put/seqid=0 2024-12-06T14:41:58,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741897_1073 (size=12983) 2024-12-06T14:41:58,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741899_1075 (size=12983) 2024-12-06T14:41:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741898_1074 (size=12301) 2024-12-06T14:41:58,965 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/8aa74fc18b464a84bb07f2b8d4b70c3a 2024-12-06T14:41:58,967 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/52697209e92240078274585eda571287 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/52697209e92240078274585eda571287 2024-12-06T14:41:58,977 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 52697209e92240078274585eda571287(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:58,977 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:58,977 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=13, startTime=1733496118923; duration=0sec 2024-12-06T14:41:58,977 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:41:58,977 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:41:58,977 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:41:58,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/cea17647a1f74d398c10e605296bd177 is 50, key is test_row_0/B:col10/1733496118030/Put/seqid=0 2024-12-06T14:41:58,980 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:41:58,980 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:41:58,980 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:58,980 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/acb44ab48c204dbba76c86adc7cefda2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f299ad024d274fc78c58eb789a3a8569, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/840c036da67c4cf1aaf57c965946324b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=36.6 K 2024-12-06T14:41:58,981 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting acb44ab48c204dbba76c86adc7cefda2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733496115624 2024-12-06T14:41:58,981 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f299ad024d274fc78c58eb789a3a8569, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496116038 2024-12-06T14:41:58,983 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840c036da67c4cf1aaf57c965946324b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733496117671 2024-12-06T14:41:59,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741900_1076 (size=12301) 2024-12-06T14:41:59,018 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#64 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:41:59,018 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/7e74ee65652543a8b39420b959aa245d is 50, key is test_row_0/C:col10/1733496118000/Put/seqid=0 2024-12-06T14:41:59,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741901_1077 (size=12983) 2024-12-06T14:41:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:59,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:41:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496179253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496179252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T14:41:59,360 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/b49ab87d88eb4987a31b05091d01dd3c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/b49ab87d88eb4987a31b05091d01dd3c 2024-12-06T14:41:59,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496179368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,373 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into b49ab87d88eb4987a31b05091d01dd3c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:59,373 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,373 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=13, startTime=1733496118923; duration=0sec 2024-12-06T14:41:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,374 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496179369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,374 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:41:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,413 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/cea17647a1f74d398c10e605296bd177 2024-12-06T14:41:59,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/ae1d2bef8a6f49be8e524548060ba4fb is 50, key is test_row_0/C:col10/1733496118030/Put/seqid=0 2024-12-06T14:41:59,440 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/7e74ee65652543a8b39420b959aa245d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7e74ee65652543a8b39420b959aa245d 2024-12-06T14:41:59,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741902_1078 (size=12301) 2024-12-06T14:41:59,459 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into 7e74ee65652543a8b39420b959aa245d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:41:59,459 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:59,459 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=13, startTime=1733496118923; duration=0sec 2024-12-06T14:41:59,460 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:41:59,460 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:41:59,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/ae1d2bef8a6f49be8e524548060ba4fb 2024-12-06T14:41:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/8aa74fc18b464a84bb07f2b8d4b70c3a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8aa74fc18b464a84bb07f2b8d4b70c3a 2024-12-06T14:41:59,492 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8aa74fc18b464a84bb07f2b8d4b70c3a, entries=150, sequenceid=372, filesize=12.0 K 2024-12-06T14:41:59,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/cea17647a1f74d398c10e605296bd177 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/cea17647a1f74d398c10e605296bd177 2024-12-06T14:41:59,502 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/cea17647a1f74d398c10e605296bd177, entries=150, sequenceid=372, filesize=12.0 K 2024-12-06T14:41:59,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/ae1d2bef8a6f49be8e524548060ba4fb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ae1d2bef8a6f49be8e524548060ba4fb 2024-12-06T14:41:59,515 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ae1d2bef8a6f49be8e524548060ba4fb, entries=150, sequenceid=372, filesize=12.0 K 2024-12-06T14:41:59,517 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 709a255a23a9b4b1363f2ec84dd61843 in 584ms, sequenceid=372, compaction requested=false 2024-12-06T14:41:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:41:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:41:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-06T14:41:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-06T14:41:59,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-06T14:41:59,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3550 sec 2024-12-06T14:41:59,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3640 sec 2024-12-06T14:41:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:41:59,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:41:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:41:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:41:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:41:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:41:59,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/4e86844ca7be43a6b9507137f9ad0520 is 50, key is test_row_0/A:col10/1733496119579/Put/seqid=0 2024-12-06T14:41:59,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741903_1079 (size=12301) 2024-12-06T14:41:59,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/4e86844ca7be43a6b9507137f9ad0520 2024-12-06T14:41:59,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/47a81aa9298547ed99b9a0ba1d42b7c0 is 50, key is test_row_0/B:col10/1733496119579/Put/seqid=0 2024-12-06T14:41:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496179675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496179678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741904_1080 (size=12301) 2024-12-06T14:41:59,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496179788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:41:59,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:41:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496179788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:41:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:41:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496179999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496179999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/47a81aa9298547ed99b9a0ba1d42b7c0 2024-12-06T14:42:00,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6d2509e5edc64f0ebc4137d81f3d9b15 is 50, key is test_row_0/C:col10/1733496119579/Put/seqid=0 2024-12-06T14:42:00,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741905_1081 (size=12301) 2024-12-06T14:42:00,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6d2509e5edc64f0ebc4137d81f3d9b15 2024-12-06T14:42:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/4e86844ca7be43a6b9507137f9ad0520 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/4e86844ca7be43a6b9507137f9ad0520 2024-12-06T14:42:00,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/4e86844ca7be43a6b9507137f9ad0520, entries=150, sequenceid=392, filesize=12.0 K 2024-12-06T14:42:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/47a81aa9298547ed99b9a0ba1d42b7c0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/47a81aa9298547ed99b9a0ba1d42b7c0 2024-12-06T14:42:00,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/47a81aa9298547ed99b9a0ba1d42b7c0, entries=150, sequenceid=392, filesize=12.0 K 2024-12-06T14:42:00,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6d2509e5edc64f0ebc4137d81f3d9b15 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6d2509e5edc64f0ebc4137d81f3d9b15 2024-12-06T14:42:00,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6d2509e5edc64f0ebc4137d81f3d9b15, entries=150, sequenceid=392, filesize=12.0 K 2024-12-06T14:42:00,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 709a255a23a9b4b1363f2ec84dd61843 in 629ms, sequenceid=392, compaction requested=true 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:00,219 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:00,219 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:00,220 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:00,220 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:00,220 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:42:00,220 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:42:00,221 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,221 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,221 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/52697209e92240078274585eda571287, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8aa74fc18b464a84bb07f2b8d4b70c3a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/4e86844ca7be43a6b9507137f9ad0520] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=36.7 K 2024-12-06T14:42:00,221 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/b49ab87d88eb4987a31b05091d01dd3c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/cea17647a1f74d398c10e605296bd177, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/47a81aa9298547ed99b9a0ba1d42b7c0] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=36.7 K 2024-12-06T14:42:00,221 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 52697209e92240078274585eda571287, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733496117671 2024-12-06T14:42:00,221 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49ab87d88eb4987a31b05091d01dd3c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733496117671 2024-12-06T14:42:00,222 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa74fc18b464a84bb07f2b8d4b70c3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733496118007 2024-12-06T14:42:00,222 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea17647a1f74d398c10e605296bd177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733496118007 2024-12-06T14:42:00,222 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47a81aa9298547ed99b9a0ba1d42b7c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496119217 2024-12-06T14:42:00,222 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e86844ca7be43a6b9507137f9ad0520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496119217 2024-12-06T14:42:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:00,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:42:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:00,237 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:00,237 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/803163df38524e52894246c3046da1f6 is 50, key is test_row_0/A:col10/1733496119579/Put/seqid=0 2024-12-06T14:42:00,242 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:00,243 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/8ca6cb5a279b4dd6b7cedd97271ec683 is 50, key is test_row_0/B:col10/1733496119579/Put/seqid=0 2024-12-06T14:42:00,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/3042ce276f744205b9ec0935321ddbb6 is 50, key is test_row_0/A:col10/1733496119641/Put/seqid=0 2024-12-06T14:42:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496180258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496180264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496180264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741906_1082 (size=13085) 2024-12-06T14:42:00,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741907_1083 (size=13085) 2024-12-06T14:42:00,302 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/8ca6cb5a279b4dd6b7cedd97271ec683 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/8ca6cb5a279b4dd6b7cedd97271ec683 2024-12-06T14:42:00,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741908_1084 (size=17181) 2024-12-06T14:42:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T14:42:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,309 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496180309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-06T14:42:00,313 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T14:42:00,313 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:00,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:00,315 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into 8ca6cb5a279b4dd6b7cedd97271ec683(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:00,315 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:00,315 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=13, startTime=1733496120219; duration=0sec 2024-12-06T14:42:00,317 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:00,317 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:42:00,317 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:00,319 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:00,319 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:42:00,319 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,319 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7e74ee65652543a8b39420b959aa245d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ae1d2bef8a6f49be8e524548060ba4fb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6d2509e5edc64f0ebc4137d81f3d9b15] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=36.7 K 2024-12-06T14:42:00,320 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e74ee65652543a8b39420b959aa245d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733496117671 2024-12-06T14:42:00,320 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1d2bef8a6f49be8e524548060ba4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733496118007 2024-12-06T14:42:00,321 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2509e5edc64f0ebc4137d81f3d9b15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496119217 2024-12-06T14:42:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496180318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,332 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:00,333 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/50196cc3d59c47ce901366af2229108c is 50, key is test_row_0/C:col10/1733496119579/Put/seqid=0 2024-12-06T14:42:00,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741909_1085 (size=13085) 2024-12-06T14:42:00,370 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/50196cc3d59c47ce901366af2229108c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/50196cc3d59c47ce901366af2229108c 2024-12-06T14:42:00,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496180369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496180380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496180381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,390 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into 50196cc3d59c47ce901366af2229108c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:00,390 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:00,390 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=13, startTime=1733496120219; duration=0sec 2024-12-06T14:42:00,391 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:00,391 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:42:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T14:42:00,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T14:42:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496180579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496180588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496180588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T14:42:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T14:42:00,681 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/803163df38524e52894246c3046da1f6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/803163df38524e52894246c3046da1f6 2024-12-06T14:42:00,695 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 803163df38524e52894246c3046da1f6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:00,695 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:00,695 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=13, startTime=1733496120219; duration=0sec 2024-12-06T14:42:00,695 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:00,695 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:42:00,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/3042ce276f744205b9ec0935321ddbb6 2024-12-06T14:42:00,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c43e138d9dc248d7baa6c195f932fbee is 50, key is test_row_0/B:col10/1733496119641/Put/seqid=0 2024-12-06T14:42:00,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741910_1086 (size=12301) 2024-12-06T14:42:00,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T14:42:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496180818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496180828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496180888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496180898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496180898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:00,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T14:42:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:00,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T14:42:01,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T14:42:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:01,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:01,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c43e138d9dc248d7baa6c195f932fbee 2024-12-06T14:42:01,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/256bc4dceca74b39a4b8cf3f28ac8c13 is 50, key is test_row_0/C:col10/1733496119641/Put/seqid=0 2024-12-06T14:42:01,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741911_1087 (size=12301) 2024-12-06T14:42:01,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/256bc4dceca74b39a4b8cf3f28ac8c13 2024-12-06T14:42:01,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/3042ce276f744205b9ec0935321ddbb6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3042ce276f744205b9ec0935321ddbb6 2024-12-06T14:42:01,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3042ce276f744205b9ec0935321ddbb6, entries=250, sequenceid=409, filesize=16.8 K 2024-12-06T14:42:01,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c43e138d9dc248d7baa6c195f932fbee as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c43e138d9dc248d7baa6c195f932fbee 2024-12-06T14:42:01,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c43e138d9dc248d7baa6c195f932fbee, entries=150, sequenceid=409, filesize=12.0 K 2024-12-06T14:42:01,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/256bc4dceca74b39a4b8cf3f28ac8c13 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/256bc4dceca74b39a4b8cf3f28ac8c13 2024-12-06T14:42:01,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/256bc4dceca74b39a4b8cf3f28ac8c13, entries=150, sequenceid=409, filesize=12.0 K 2024-12-06T14:42:01,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 709a255a23a9b4b1363f2ec84dd61843 in 949ms, sequenceid=409, compaction requested=false 2024-12-06T14:42:01,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:01,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T14:42:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:01,232 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T14:42:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:01,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/48de0c8cd4d04575a3e64558a0f497eb is 50, key is test_row_0/A:col10/1733496120261/Put/seqid=0 2024-12-06T14:42:01,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741912_1088 (size=12301) 2024-12-06T14:42:01,241 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/48de0c8cd4d04575a3e64558a0f497eb 2024-12-06T14:42:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/2d1883b5b1fc4823bbf2592aad7a1225 is 50, key is test_row_0/B:col10/1733496120261/Put/seqid=0 2024-12-06T14:42:01,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741913_1089 (size=12301) 2024-12-06T14:42:01,254 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/2d1883b5b1fc4823bbf2592aad7a1225 2024-12-06T14:42:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5185b62126f64c0cb88232dc3ba19a87 is 50, key is test_row_0/C:col10/1733496120261/Put/seqid=0 2024-12-06T14:42:01,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741914_1090 (size=12301) 2024-12-06T14:42:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496181424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496181427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496181427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T14:42:01,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496181528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496181538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496181538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,669 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5185b62126f64c0cb88232dc3ba19a87 2024-12-06T14:42:01,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/48de0c8cd4d04575a3e64558a0f497eb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/48de0c8cd4d04575a3e64558a0f497eb 2024-12-06T14:42:01,684 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/48de0c8cd4d04575a3e64558a0f497eb, entries=150, sequenceid=428, filesize=12.0 K 2024-12-06T14:42:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/2d1883b5b1fc4823bbf2592aad7a1225 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/2d1883b5b1fc4823bbf2592aad7a1225 2024-12-06T14:42:01,692 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/2d1883b5b1fc4823bbf2592aad7a1225, entries=150, sequenceid=428, filesize=12.0 K 2024-12-06T14:42:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5185b62126f64c0cb88232dc3ba19a87 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5185b62126f64c0cb88232dc3ba19a87 2024-12-06T14:42:01,702 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5185b62126f64c0cb88232dc3ba19a87, entries=150, sequenceid=428, filesize=12.0 K 2024-12-06T14:42:01,703 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 709a255a23a9b4b1363f2ec84dd61843 in 471ms, sequenceid=428, compaction requested=true 2024-12-06T14:42:01,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:01,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-06T14:42:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-06T14:42:01,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-06T14:42:01,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3910 sec 2024-12-06T14:42:01,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3970 sec 2024-12-06T14:42:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:01,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:01,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/efc88c4f87b245088cf17199085a2fb2 is 50, key is test_row_0/A:col10/1733496121423/Put/seqid=0 2024-12-06T14:42:01,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741915_1091 (size=14741) 2024-12-06T14:42:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496181789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496181791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496181792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496181828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,830 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496181848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,852 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496181898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496181898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496181898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496182108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496182108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496182108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/efc88c4f87b245088cf17199085a2fb2 2024-12-06T14:42:02,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0276cad6b6004c2f9525206df31ac290 is 50, key is test_row_0/B:col10/1733496121423/Put/seqid=0 2024-12-06T14:42:02,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741916_1092 (size=12301) 2024-12-06T14:42:02,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0276cad6b6004c2f9525206df31ac290 2024-12-06T14:42:02,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/30d7dda52aff4d65b1127dd80bbf1acc is 50, key is test_row_0/C:col10/1733496121423/Put/seqid=0 2024-12-06T14:42:02,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741917_1093 (size=12301) 2024-12-06T14:42:02,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/30d7dda52aff4d65b1127dd80bbf1acc 2024-12-06T14:42:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/efc88c4f87b245088cf17199085a2fb2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/efc88c4f87b245088cf17199085a2fb2 2024-12-06T14:42:02,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/efc88c4f87b245088cf17199085a2fb2, entries=200, sequenceid=446, filesize=14.4 K 2024-12-06T14:42:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0276cad6b6004c2f9525206df31ac290 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0276cad6b6004c2f9525206df31ac290 2024-12-06T14:42:02,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0276cad6b6004c2f9525206df31ac290, entries=150, sequenceid=446, filesize=12.0 K 2024-12-06T14:42:02,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/30d7dda52aff4d65b1127dd80bbf1acc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/30d7dda52aff4d65b1127dd80bbf1acc 2024-12-06T14:42:02,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/30d7dda52aff4d65b1127dd80bbf1acc, entries=150, sequenceid=446, filesize=12.0 K 2024-12-06T14:42:02,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 709a255a23a9b4b1363f2ec84dd61843 in 464ms, sequenceid=446, compaction requested=true 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:02,205 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:02,205 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:02,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:02,206 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:02,206 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:42:02,206 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,207 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/803163df38524e52894246c3046da1f6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3042ce276f744205b9ec0935321ddbb6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/48de0c8cd4d04575a3e64558a0f497eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/efc88c4f87b245088cf17199085a2fb2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=56.0 K 2024-12-06T14:42:02,207 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803163df38524e52894246c3046da1f6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496119217 2024-12-06T14:42:02,207 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:02,207 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:42:02,207 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,207 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/8ca6cb5a279b4dd6b7cedd97271ec683, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c43e138d9dc248d7baa6c195f932fbee, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/2d1883b5b1fc4823bbf2592aad7a1225, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0276cad6b6004c2f9525206df31ac290] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=48.8 K 2024-12-06T14:42:02,207 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3042ce276f744205b9ec0935321ddbb6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733496119641 2024-12-06T14:42:02,208 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca6cb5a279b4dd6b7cedd97271ec683, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496119217 2024-12-06T14:42:02,208 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48de0c8cd4d04575a3e64558a0f497eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733496120255 2024-12-06T14:42:02,208 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c43e138d9dc248d7baa6c195f932fbee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733496119641 2024-12-06T14:42:02,208 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc88c4f87b245088cf17199085a2fb2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733496121417 2024-12-06T14:42:02,209 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1883b5b1fc4823bbf2592aad7a1225, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733496120255 2024-12-06T14:42:02,209 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0276cad6b6004c2f9525206df31ac290, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733496121417 2024-12-06T14:42:02,221 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:02,221 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:02,222 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/03d744bc4e3547abb7df729217c0af8b is 50, key is test_row_0/B:col10/1733496121423/Put/seqid=0 2024-12-06T14:42:02,222 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/2c2d853b375f45ccb6b890df583ceae8 is 50, key is test_row_0/A:col10/1733496121423/Put/seqid=0 2024-12-06T14:42:02,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741919_1095 (size=13221) 2024-12-06T14:42:02,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741918_1094 (size=13221) 2024-12-06T14:42:02,248 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/2c2d853b375f45ccb6b890df583ceae8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/2c2d853b375f45ccb6b890df583ceae8 2024-12-06T14:42:02,256 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into 2c2d853b375f45ccb6b890df583ceae8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:02,256 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:02,256 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=12, startTime=1733496122205; duration=0sec 2024-12-06T14:42:02,256 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:02,256 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:42:02,257 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:02,258 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:02,259 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:42:02,259 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,259 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/50196cc3d59c47ce901366af2229108c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/256bc4dceca74b39a4b8cf3f28ac8c13, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5185b62126f64c0cb88232dc3ba19a87, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/30d7dda52aff4d65b1127dd80bbf1acc] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=48.8 K 2024-12-06T14:42:02,259 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50196cc3d59c47ce901366af2229108c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496119217 2024-12-06T14:42:02,260 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 256bc4dceca74b39a4b8cf3f28ac8c13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733496119641 2024-12-06T14:42:02,260 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5185b62126f64c0cb88232dc3ba19a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733496120255 2024-12-06T14:42:02,260 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d7dda52aff4d65b1127dd80bbf1acc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733496121417 2024-12-06T14:42:02,269 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:02,270 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f3db989e8d4a49388ebd7882599f749f is 50, key is test_row_0/C:col10/1733496121423/Put/seqid=0 2024-12-06T14:42:02,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741920_1096 (size=13221) 2024-12-06T14:42:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:02,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:42:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:02,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/39859d3a3f88494a8d4fece6c864c033 is 50, key is test_row_0/A:col10/1733496122420/Put/seqid=0 2024-12-06T14:42:02,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741921_1097 (size=14741) 2024-12-06T14:42:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T14:42:02,458 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:02,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-06T14:42:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:02,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:02,461 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:02,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:02,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496182466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496182466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496182467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:02,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496182578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496182578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496182578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,647 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/03d744bc4e3547abb7df729217c0af8b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/03d744bc4e3547abb7df729217c0af8b 2024-12-06T14:42:02,654 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into 03d744bc4e3547abb7df729217c0af8b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:02,654 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:02,654 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=12, startTime=1733496122205; duration=0sec 2024-12-06T14:42:02,654 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:02,654 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:42:02,696 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f3db989e8d4a49388ebd7882599f749f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f3db989e8d4a49388ebd7882599f749f 2024-12-06T14:42:02,702 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into f3db989e8d4a49388ebd7882599f749f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:02,702 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:02,702 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=12, startTime=1733496122205; duration=0sec 2024-12-06T14:42:02,702 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:02,702 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:42:02,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:02,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496182788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496182788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496182788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:02,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/39859d3a3f88494a8d4fece6c864c033 2024-12-06T14:42:02,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/87bc282ecf1b4a26b1e86f6d359371fe is 50, key is test_row_0/B:col10/1733496122420/Put/seqid=0 2024-12-06T14:42:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741922_1098 (size=12301) 2024-12-06T14:42:02,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:02,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496183098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496183099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496183099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/87bc282ecf1b4a26b1e86f6d359371fe 2024-12-06T14:42:03,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5c3cc1a6891546c2952cba68ba9f03fa is 50, key is test_row_0/C:col10/1733496122420/Put/seqid=0 2024-12-06T14:42:03,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741923_1099 (size=12301) 2024-12-06T14:42:03,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:03,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:03,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496183609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496183609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496183609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5c3cc1a6891546c2952cba68ba9f03fa 2024-12-06T14:42:03,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/39859d3a3f88494a8d4fece6c864c033 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/39859d3a3f88494a8d4fece6c864c033 2024-12-06T14:42:03,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/39859d3a3f88494a8d4fece6c864c033, entries=200, sequenceid=463, filesize=14.4 K 2024-12-06T14:42:03,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/87bc282ecf1b4a26b1e86f6d359371fe as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/87bc282ecf1b4a26b1e86f6d359371fe 2024-12-06T14:42:03,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:03,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/87bc282ecf1b4a26b1e86f6d359371fe, entries=150, sequenceid=463, filesize=12.0 K 2024-12-06T14:42:03,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5c3cc1a6891546c2952cba68ba9f03fa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c3cc1a6891546c2952cba68ba9f03fa 2024-12-06T14:42:03,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c3cc1a6891546c2952cba68ba9f03fa, entries=150, sequenceid=463, filesize=12.0 K 2024-12-06T14:42:03,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 709a255a23a9b4b1363f2ec84dd61843 in 1277ms, sequenceid=463, compaction requested=false 2024-12-06T14:42:03,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:03,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T14:42:03,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:03,837 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T14:42:03,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:03,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:03,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:03,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/41dc32c722214b55ae15336028617d4e is 50, key is test_row_0/A:col10/1733496122466/Put/seqid=0 2024-12-06T14:42:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:03,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. as already flushing 2024-12-06T14:42:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741924_1100 (size=12301) 2024-12-06T14:42:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496183877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:03,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496183880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496183988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496184001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496184208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496184208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,254 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/41dc32c722214b55ae15336028617d4e 2024-12-06T14:42:04,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/204e6e7c9bb7456da8f5ce0b0301afe8 is 50, key is test_row_0/B:col10/1733496122466/Put/seqid=0 2024-12-06T14:42:04,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741925_1101 (size=12301) 2024-12-06T14:42:04,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496184518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496184518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:04,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496184628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496184628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496184628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:04,631 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:04,632 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,633 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:04,668 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/204e6e7c9bb7456da8f5ce0b0301afe8 2024-12-06T14:42:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5dc89e6fbb27413ea4416ce3e179b053 is 50, key is test_row_0/C:col10/1733496122466/Put/seqid=0 2024-12-06T14:42:04,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741926_1102 (size=12301) 2024-12-06T14:42:05,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496185028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496185028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:05,088 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5dc89e6fbb27413ea4416ce3e179b053 2024-12-06T14:42:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/41dc32c722214b55ae15336028617d4e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/41dc32c722214b55ae15336028617d4e 2024-12-06T14:42:05,097 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/41dc32c722214b55ae15336028617d4e, entries=150, sequenceid=481, filesize=12.0 K 2024-12-06T14:42:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/204e6e7c9bb7456da8f5ce0b0301afe8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/204e6e7c9bb7456da8f5ce0b0301afe8 2024-12-06T14:42:05,105 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/204e6e7c9bb7456da8f5ce0b0301afe8, entries=150, sequenceid=481, filesize=12.0 K 2024-12-06T14:42:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5dc89e6fbb27413ea4416ce3e179b053 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5dc89e6fbb27413ea4416ce3e179b053 2024-12-06T14:42:05,113 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5dc89e6fbb27413ea4416ce3e179b053, entries=150, sequenceid=481, filesize=12.0 K 2024-12-06T14:42:05,114 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 709a255a23a9b4b1363f2ec84dd61843 in 1277ms, sequenceid=481, compaction requested=true 2024-12-06T14:42:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:05,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:05,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-06T14:42:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-06T14:42:05,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-06T14:42:05,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6550 sec 2024-12-06T14:42:05,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.6590 sec 2024-12-06T14:42:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:06,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:42:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:06,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/341bc8f46f19407ba1bad5c6e6cfe397 is 50, key is test_row_0/A:col10/1733496123875/Put/seqid=0 2024-12-06T14:42:06,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741927_1103 (size=14741) 2024-12-06T14:42:06,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496186063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496186064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496186168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496186168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,247 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-06T14:42:06,247 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:06,250 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-06T14:42:06,251 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:06,251 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-06T14:42:06,251 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:06,251 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-06T14:42:06,251 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:06,252 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,252 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,252 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,252 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,253 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,253 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,253 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,253 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:06,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:06,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:06,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:06,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:06,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496186379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496186379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/341bc8f46f19407ba1bad5c6e6cfe397 2024-12-06T14:42:06,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1963090d37ab460dbb910c949c376cf7 is 50, key is test_row_0/B:col10/1733496123875/Put/seqid=0 2024-12-06T14:42:06,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741928_1104 (size=12301) 2024-12-06T14:42:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T14:42:06,621 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48998 deadline: 1733496186638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,639 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49008 deadline: 1733496186648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48988 deadline: 1733496186648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,649 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:06,649 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 709a255a23a9b4b1363f2ec84dd61843 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496186688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496186689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:06,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1963090d37ab460dbb910c949c376cf7 2024-12-06T14:42:06,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/13deb4a325224761bc1c0271f0218942 is 50, key is test_row_0/C:col10/1733496123875/Put/seqid=0 2024-12-06T14:42:06,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741929_1105 (size=12301) 2024-12-06T14:42:07,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:07,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48994 deadline: 1733496187199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49018 deadline: 1733496187199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:07,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/13deb4a325224761bc1c0271f0218942 2024-12-06T14:42:07,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/341bc8f46f19407ba1bad5c6e6cfe397 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/341bc8f46f19407ba1bad5c6e6cfe397 2024-12-06T14:42:07,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/341bc8f46f19407ba1bad5c6e6cfe397, entries=200, sequenceid=498, filesize=14.4 K 2024-12-06T14:42:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/1963090d37ab460dbb910c949c376cf7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1963090d37ab460dbb910c949c376cf7 2024-12-06T14:42:07,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1963090d37ab460dbb910c949c376cf7, entries=150, sequenceid=498, filesize=12.0 K 2024-12-06T14:42:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/13deb4a325224761bc1c0271f0218942 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/13deb4a325224761bc1c0271f0218942 2024-12-06T14:42:07,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/13deb4a325224761bc1c0271f0218942, entries=150, sequenceid=498, filesize=12.0 K 2024-12-06T14:42:07,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 709a255a23a9b4b1363f2ec84dd61843 in 1280ms, sequenceid=498, compaction requested=true 2024-12-06T14:42:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:07,320 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:07,320 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 709a255a23a9b4b1363f2ec84dd61843:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:07,322 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:07,322 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:07,322 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/A is initiating minor compaction (all files) 2024-12-06T14:42:07,322 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/B is initiating minor compaction (all files) 2024-12-06T14:42:07,322 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/B in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:07,322 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/A in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:07,322 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/2c2d853b375f45ccb6b890df583ceae8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/39859d3a3f88494a8d4fece6c864c033, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/41dc32c722214b55ae15336028617d4e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/341bc8f46f19407ba1bad5c6e6cfe397] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=53.7 K 2024-12-06T14:42:07,322 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/03d744bc4e3547abb7df729217c0af8b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/87bc282ecf1b4a26b1e86f6d359371fe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/204e6e7c9bb7456da8f5ce0b0301afe8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1963090d37ab460dbb910c949c376cf7] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=48.9 K 2024-12-06T14:42:07,323 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c2d853b375f45ccb6b890df583ceae8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733496121417 2024-12-06T14:42:07,323 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d744bc4e3547abb7df729217c0af8b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733496121417 2024-12-06T14:42:07,323 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39859d3a3f88494a8d4fece6c864c033, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733496121764 2024-12-06T14:42:07,323 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bc282ecf1b4a26b1e86f6d359371fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733496121790 2024-12-06T14:42:07,323 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41dc32c722214b55ae15336028617d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733496122465 2024-12-06T14:42:07,323 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 204e6e7c9bb7456da8f5ce0b0301afe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733496122465 2024-12-06T14:42:07,324 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 341bc8f46f19407ba1bad5c6e6cfe397, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733496123873 2024-12-06T14:42:07,324 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1963090d37ab460dbb910c949c376cf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733496123873 2024-12-06T14:42:07,333 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:07,334 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/c7220d0c846449e5a3be595dd0679da7 is 50, key is test_row_0/A:col10/1733496123875/Put/seqid=0 2024-12-06T14:42:07,334 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:07,335 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c0ab52a2e9c647a8a0e6fc2fb40ac1e9 is 50, key is test_row_0/B:col10/1733496123875/Put/seqid=0 2024-12-06T14:42:07,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741930_1106 (size=13357) 2024-12-06T14:42:07,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741931_1107 (size=13357) 2024-12-06T14:42:07,752 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/c7220d0c846449e5a3be595dd0679da7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c7220d0c846449e5a3be595dd0679da7 2024-12-06T14:42:07,753 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/c0ab52a2e9c647a8a0e6fc2fb40ac1e9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0ab52a2e9c647a8a0e6fc2fb40ac1e9 2024-12-06T14:42:07,759 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/A of 709a255a23a9b4b1363f2ec84dd61843 into c7220d0c846449e5a3be595dd0679da7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:07,759 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/B of 709a255a23a9b4b1363f2ec84dd61843 into c0ab52a2e9c647a8a0e6fc2fb40ac1e9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:07,759 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/B, priority=12, startTime=1733496127320; duration=0sec 2024-12-06T14:42:07,759 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/A, priority=12, startTime=1733496127320; duration=0sec 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:B 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:A 2024-12-06T14:42:07,759 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:07,761 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:07,761 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 709a255a23a9b4b1363f2ec84dd61843/C is initiating minor compaction (all files) 2024-12-06T14:42:07,761 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 709a255a23a9b4b1363f2ec84dd61843/C in TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:07,761 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f3db989e8d4a49388ebd7882599f749f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c3cc1a6891546c2952cba68ba9f03fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5dc89e6fbb27413ea4416ce3e179b053, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/13deb4a325224761bc1c0271f0218942] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp, totalSize=48.9 K 2024-12-06T14:42:07,761 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f3db989e8d4a49388ebd7882599f749f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733496121417 2024-12-06T14:42:07,762 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3cc1a6891546c2952cba68ba9f03fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733496121790 2024-12-06T14:42:07,762 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc89e6fbb27413ea4416ce3e179b053, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733496122465 2024-12-06T14:42:07,762 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 13deb4a325224761bc1c0271f0218942, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733496123873 2024-12-06T14:42:07,771 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 709a255a23a9b4b1363f2ec84dd61843#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:07,771 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5c0251fdf71a48ae9eb4030cb648ff0d is 50, key is test_row_0/C:col10/1733496123875/Put/seqid=0 2024-12-06T14:42:07,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741932_1108 (size=13357) 2024-12-06T14:42:08,185 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/5c0251fdf71a48ae9eb4030cb648ff0d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c0251fdf71a48ae9eb4030cb648ff0d 2024-12-06T14:42:08,191 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 709a255a23a9b4b1363f2ec84dd61843/C of 709a255a23a9b4b1363f2ec84dd61843 into 5c0251fdf71a48ae9eb4030cb648ff0d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:08,191 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:08,191 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., storeName=709a255a23a9b4b1363f2ec84dd61843/C, priority=12, startTime=1733496127321; duration=0sec 2024-12-06T14:42:08,191 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:08,192 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 709a255a23a9b4b1363f2ec84dd61843:C 2024-12-06T14:42:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:08,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:42:08,211 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-06T14:42:08,211 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:08,211 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-06T14:42:08,212 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:08,212 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:08,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:08,212 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:08,212 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:08,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:08,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:08,212 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:08,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:08,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:08,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:08,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:08,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:08,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/feace0a9aa1f473889a5e16aece49ec0 is 50, key is test_row_0/A:col10/1733496126056/Put/seqid=0 2024-12-06T14:42:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741933_1109 (size=12301) 2024-12-06T14:42:08,546 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T14:42:08,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/feace0a9aa1f473889a5e16aece49ec0 2024-12-06T14:42:08,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0ff1fd8b23dd444487db6a30b11d5168 is 50, key is test_row_0/B:col10/1733496126056/Put/seqid=0 2024-12-06T14:42:08,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741934_1110 (size=12301) 2024-12-06T14:42:09,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0ff1fd8b23dd444487db6a30b11d5168 2024-12-06T14:42:09,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6ba5dd74eab24c8087384d6766dde731 is 50, key is test_row_0/C:col10/1733496126056/Put/seqid=0 2024-12-06T14:42:09,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741935_1111 (size=12301) 2024-12-06T14:42:09,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6ba5dd74eab24c8087384d6766dde731 2024-12-06T14:42:09,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/feace0a9aa1f473889a5e16aece49ec0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/feace0a9aa1f473889a5e16aece49ec0 2024-12-06T14:42:09,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/feace0a9aa1f473889a5e16aece49ec0, entries=150, sequenceid=519, filesize=12.0 K 2024-12-06T14:42:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/0ff1fd8b23dd444487db6a30b11d5168 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0ff1fd8b23dd444487db6a30b11d5168 2024-12-06T14:42:09,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0ff1fd8b23dd444487db6a30b11d5168, entries=150, sequenceid=519, filesize=12.0 K 2024-12-06T14:42:09,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/6ba5dd74eab24c8087384d6766dde731 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6ba5dd74eab24c8087384d6766dde731 2024-12-06T14:42:09,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6ba5dd74eab24c8087384d6766dde731, entries=150, sequenceid=519, filesize=12.0 K 2024-12-06T14:42:09,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 709a255a23a9b4b1363f2ec84dd61843 in 1287ms, sequenceid=519, compaction requested=false 2024-12-06T14:42:09,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 709a255a23a9b4b1363f2ec84dd61843: 2024-12-06T14:42:10,650 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-06T14:42:10,650 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:10,650 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,650 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:10,659 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-06T14:42:10,660 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:10,660 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,660 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:10,681 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-06T14:42:10,681 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:10,681 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,681 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:10,681 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3815 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3561 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T14:42:10,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1625 2024-12-06T14:42:10,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4871 rows 2024-12-06T14:42:10,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1622 2024-12-06T14:42:10,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4865 rows 2024-12-06T14:42:10,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:42:10,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:42:10,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:10,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:10,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T14:42:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T14:42:10,701 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496130701"}]},"ts":"1733496130701"} 2024-12-06T14:42:10,704 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T14:42:10,704 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T14:42:10,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:42:10,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, UNASSIGN}] 2024-12-06T14:42:10,711 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, UNASSIGN 2024-12-06T14:42:10,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=709a255a23a9b4b1363f2ec84dd61843, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:10,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, UNASSIGN because future has completed 2024-12-06T14:42:10,715 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:42:10,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:42:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T14:42:10,875 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:10,875 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:42:10,877 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 709a255a23a9b4b1363f2ec84dd61843, disabling compactions & flushes 2024-12-06T14:42:10,877 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:10,877 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:10,877 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. after waiting 0 ms 2024-12-06T14:42:10,877 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:10,877 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 709a255a23a9b4b1363f2ec84dd61843 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-06T14:42:10,878 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=A 2024-12-06T14:42:10,878 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:10,878 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=B 2024-12-06T14:42:10,878 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:10,878 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 709a255a23a9b4b1363f2ec84dd61843, store=C 2024-12-06T14:42:10,878 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:10,885 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/b193ab5b943f483485bae48c8a481ada is 50, key is test_row_0/A:col10/1733496130649/Put/seqid=0 2024-12-06T14:42:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741936_1112 (size=9857) 2024-12-06T14:42:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T14:42:11,290 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/b193ab5b943f483485bae48c8a481ada 2024-12-06T14:42:11,297 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/9e535ebf0ba64ed7881cffc6bb827482 is 50, key is test_row_0/B:col10/1733496130649/Put/seqid=0 2024-12-06T14:42:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741937_1113 (size=9857) 2024-12-06T14:42:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T14:42:11,702 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/9e535ebf0ba64ed7881cffc6bb827482 2024-12-06T14:42:11,713 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f456a9915b59479c921be4cfef3fdaa7 is 50, key is test_row_0/C:col10/1733496130649/Put/seqid=0 2024-12-06T14:42:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741938_1114 (size=9857) 2024-12-06T14:42:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T14:42:12,119 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f456a9915b59479c921be4cfef3fdaa7 2024-12-06T14:42:12,125 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/A/b193ab5b943f483485bae48c8a481ada as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/b193ab5b943f483485bae48c8a481ada 2024-12-06T14:42:12,130 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/b193ab5b943f483485bae48c8a481ada, entries=100, sequenceid=525, filesize=9.6 K 2024-12-06T14:42:12,131 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/B/9e535ebf0ba64ed7881cffc6bb827482 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/9e535ebf0ba64ed7881cffc6bb827482 2024-12-06T14:42:12,136 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/9e535ebf0ba64ed7881cffc6bb827482, entries=100, sequenceid=525, filesize=9.6 K 2024-12-06T14:42:12,137 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/.tmp/C/f456a9915b59479c921be4cfef3fdaa7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f456a9915b59479c921be4cfef3fdaa7 2024-12-06T14:42:12,141 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f456a9915b59479c921be4cfef3fdaa7, entries=100, sequenceid=525, filesize=9.6 K 2024-12-06T14:42:12,142 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 709a255a23a9b4b1363f2ec84dd61843 in 1265ms, sequenceid=525, compaction requested=true 2024-12-06T14:42:12,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/147a93cbc49f42ba8679af9297dba5f1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/321ad75928f74ce8ae3a0a176e4dd0b4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3fa8e0bfe4fa4e62ad01c080aa070b20, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/35841d56c27c4b76966d9791fa730cd3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/0c57f59affff4671acc7a4a7b900a3ed, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/434d22d1e59f49a7b026d4642bf84cf8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/18ff973c07154165b44e181c0f50a15d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/94f4fced348f4844ad9c67019bfaa26c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/ac2d513a8b2f4eec8ed0e783ded7bfee, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/f1ec2500700d4ef093ceca0c311249d6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/50fadea7a9ed43f5b0663662decd4105, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/915788bc52e54f1c9f980f4dec83fc08, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6719bb0589074bb28678eda6b440a09c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6bc9ac7034f74e86a65132b8cad93091, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/760d0fc260c442b185fbba842d644c91, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8d188a90570b4810b4e8f8ca58cf074f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/33a21a42f8b94c3db7f65f267a9df2bf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/e24d6567057047beb98a59de00739cdd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c0eade68238a40f2b02b3043f1c8b8d9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/100d24687f5c44348de6ef1792d7314d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/52697209e92240078274585eda571287, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8aa74fc18b464a84bb07f2b8d4b70c3a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/803163df38524e52894246c3046da1f6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/4e86844ca7be43a6b9507137f9ad0520, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3042ce276f744205b9ec0935321ddbb6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/48de0c8cd4d04575a3e64558a0f497eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/efc88c4f87b245088cf17199085a2fb2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/2c2d853b375f45ccb6b890df583ceae8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/39859d3a3f88494a8d4fece6c864c033, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/41dc32c722214b55ae15336028617d4e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/341bc8f46f19407ba1bad5c6e6cfe397] to archive 2024-12-06T14:42:12,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:42:12,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/147a93cbc49f42ba8679af9297dba5f1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/147a93cbc49f42ba8679af9297dba5f1 2024-12-06T14:42:12,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/321ad75928f74ce8ae3a0a176e4dd0b4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/321ad75928f74ce8ae3a0a176e4dd0b4 2024-12-06T14:42:12,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3fa8e0bfe4fa4e62ad01c080aa070b20 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3fa8e0bfe4fa4e62ad01c080aa070b20 2024-12-06T14:42:12,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/35841d56c27c4b76966d9791fa730cd3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/35841d56c27c4b76966d9791fa730cd3 2024-12-06T14:42:12,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/0c57f59affff4671acc7a4a7b900a3ed to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/0c57f59affff4671acc7a4a7b900a3ed 2024-12-06T14:42:12,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/434d22d1e59f49a7b026d4642bf84cf8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/434d22d1e59f49a7b026d4642bf84cf8 2024-12-06T14:42:12,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/18ff973c07154165b44e181c0f50a15d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/18ff973c07154165b44e181c0f50a15d 2024-12-06T14:42:12,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/94f4fced348f4844ad9c67019bfaa26c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/94f4fced348f4844ad9c67019bfaa26c 2024-12-06T14:42:12,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/ac2d513a8b2f4eec8ed0e783ded7bfee to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/ac2d513a8b2f4eec8ed0e783ded7bfee 2024-12-06T14:42:12,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/f1ec2500700d4ef093ceca0c311249d6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/f1ec2500700d4ef093ceca0c311249d6 2024-12-06T14:42:12,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/50fadea7a9ed43f5b0663662decd4105 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/50fadea7a9ed43f5b0663662decd4105 2024-12-06T14:42:12,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/915788bc52e54f1c9f980f4dec83fc08 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/915788bc52e54f1c9f980f4dec83fc08 2024-12-06T14:42:12,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6719bb0589074bb28678eda6b440a09c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6719bb0589074bb28678eda6b440a09c 2024-12-06T14:42:12,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6bc9ac7034f74e86a65132b8cad93091 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/6bc9ac7034f74e86a65132b8cad93091 2024-12-06T14:42:12,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/760d0fc260c442b185fbba842d644c91 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/760d0fc260c442b185fbba842d644c91 2024-12-06T14:42:12,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8d188a90570b4810b4e8f8ca58cf074f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8d188a90570b4810b4e8f8ca58cf074f 2024-12-06T14:42:12,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/33a21a42f8b94c3db7f65f267a9df2bf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/33a21a42f8b94c3db7f65f267a9df2bf 2024-12-06T14:42:12,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/e24d6567057047beb98a59de00739cdd to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/e24d6567057047beb98a59de00739cdd 2024-12-06T14:42:12,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c0eade68238a40f2b02b3043f1c8b8d9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c0eade68238a40f2b02b3043f1c8b8d9 2024-12-06T14:42:12,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/100d24687f5c44348de6ef1792d7314d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/100d24687f5c44348de6ef1792d7314d 2024-12-06T14:42:12,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/52697209e92240078274585eda571287 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/52697209e92240078274585eda571287 2024-12-06T14:42:12,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8aa74fc18b464a84bb07f2b8d4b70c3a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/8aa74fc18b464a84bb07f2b8d4b70c3a 2024-12-06T14:42:12,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/803163df38524e52894246c3046da1f6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/803163df38524e52894246c3046da1f6 2024-12-06T14:42:12,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/4e86844ca7be43a6b9507137f9ad0520 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/4e86844ca7be43a6b9507137f9ad0520 2024-12-06T14:42:12,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3042ce276f744205b9ec0935321ddbb6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/3042ce276f744205b9ec0935321ddbb6 2024-12-06T14:42:12,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/48de0c8cd4d04575a3e64558a0f497eb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/48de0c8cd4d04575a3e64558a0f497eb 2024-12-06T14:42:12,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/efc88c4f87b245088cf17199085a2fb2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/efc88c4f87b245088cf17199085a2fb2 2024-12-06T14:42:12,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/2c2d853b375f45ccb6b890df583ceae8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/2c2d853b375f45ccb6b890df583ceae8 2024-12-06T14:42:12,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/39859d3a3f88494a8d4fece6c864c033 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/39859d3a3f88494a8d4fece6c864c033 2024-12-06T14:42:12,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/41dc32c722214b55ae15336028617d4e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/41dc32c722214b55ae15336028617d4e 2024-12-06T14:42:12,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/341bc8f46f19407ba1bad5c6e6cfe397 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/341bc8f46f19407ba1bad5c6e6cfe397 2024-12-06T14:42:12,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/052abd5e864245e4956cc6dbe2b46434, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1ab35e9602644d36abc0b8b7af523ec6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/fe737ad3b57643778e988f284b5e0253, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/d9b6490e9c4e4978881ade98bd148c46, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/bfc10881f7284ad4a29471cba534a962, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/e52fd5c8fcfe460489c48448ec95c329, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0d2bf879b3243e3bd1c5122bf3e3776, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1c12d93e9be84cd8a4b05a3f7a4be899, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/a14b307478f64d6d81e898296a7ed983, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/90d3501e4f284704893450d112f28c6c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f14b17be82a9443a8430e7535af8bde2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/be392f6fadba48219abe4718c29c4cf2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f04d11e0d32147c5971a1d740a5cfae5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/7108fa419dbf4d9e9b53ccdcbfc3316c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/43c886282b294947b726ce949a295021, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/ce89ade3144c4c859a23ea411036eb21, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/36599200808d487dba769548980d6273, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0a84a99425b04f4c95ebfb456f22e23d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/768af12189da4915b3536bdfa2fdbd6e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/b49ab87d88eb4987a31b05091d01dd3c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/93a0309900d44d9e9205cc2100cd3fa8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/cea17647a1f74d398c10e605296bd177, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/8ca6cb5a279b4dd6b7cedd97271ec683, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/47a81aa9298547ed99b9a0ba1d42b7c0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c43e138d9dc248d7baa6c195f932fbee, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/2d1883b5b1fc4823bbf2592aad7a1225, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/03d744bc4e3547abb7df729217c0af8b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0276cad6b6004c2f9525206df31ac290, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/87bc282ecf1b4a26b1e86f6d359371fe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/204e6e7c9bb7456da8f5ce0b0301afe8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1963090d37ab460dbb910c949c376cf7] to archive 2024-12-06T14:42:12,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:42:12,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/052abd5e864245e4956cc6dbe2b46434 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/052abd5e864245e4956cc6dbe2b46434 2024-12-06T14:42:12,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1ab35e9602644d36abc0b8b7af523ec6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1ab35e9602644d36abc0b8b7af523ec6 2024-12-06T14:42:12,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/fe737ad3b57643778e988f284b5e0253 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/fe737ad3b57643778e988f284b5e0253 2024-12-06T14:42:12,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/d9b6490e9c4e4978881ade98bd148c46 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/d9b6490e9c4e4978881ade98bd148c46 2024-12-06T14:42:12,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/bfc10881f7284ad4a29471cba534a962 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/bfc10881f7284ad4a29471cba534a962 2024-12-06T14:42:12,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/e52fd5c8fcfe460489c48448ec95c329 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/e52fd5c8fcfe460489c48448ec95c329 2024-12-06T14:42:12,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0d2bf879b3243e3bd1c5122bf3e3776 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0d2bf879b3243e3bd1c5122bf3e3776 2024-12-06T14:42:12,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1c12d93e9be84cd8a4b05a3f7a4be899 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1c12d93e9be84cd8a4b05a3f7a4be899 2024-12-06T14:42:12,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/a14b307478f64d6d81e898296a7ed983 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/a14b307478f64d6d81e898296a7ed983 2024-12-06T14:42:12,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/90d3501e4f284704893450d112f28c6c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/90d3501e4f284704893450d112f28c6c 2024-12-06T14:42:12,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f14b17be82a9443a8430e7535af8bde2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f14b17be82a9443a8430e7535af8bde2 2024-12-06T14:42:12,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/be392f6fadba48219abe4718c29c4cf2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/be392f6fadba48219abe4718c29c4cf2 2024-12-06T14:42:12,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f04d11e0d32147c5971a1d740a5cfae5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/f04d11e0d32147c5971a1d740a5cfae5 2024-12-06T14:42:12,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/7108fa419dbf4d9e9b53ccdcbfc3316c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/7108fa419dbf4d9e9b53ccdcbfc3316c 2024-12-06T14:42:12,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/43c886282b294947b726ce949a295021 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/43c886282b294947b726ce949a295021 2024-12-06T14:42:12,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/ce89ade3144c4c859a23ea411036eb21 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/ce89ade3144c4c859a23ea411036eb21 2024-12-06T14:42:12,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/36599200808d487dba769548980d6273 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/36599200808d487dba769548980d6273 2024-12-06T14:42:12,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0a84a99425b04f4c95ebfb456f22e23d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0a84a99425b04f4c95ebfb456f22e23d 2024-12-06T14:42:12,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/768af12189da4915b3536bdfa2fdbd6e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/768af12189da4915b3536bdfa2fdbd6e 2024-12-06T14:42:12,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/b49ab87d88eb4987a31b05091d01dd3c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/b49ab87d88eb4987a31b05091d01dd3c 2024-12-06T14:42:12,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/93a0309900d44d9e9205cc2100cd3fa8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/93a0309900d44d9e9205cc2100cd3fa8 2024-12-06T14:42:12,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/cea17647a1f74d398c10e605296bd177 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/cea17647a1f74d398c10e605296bd177 2024-12-06T14:42:12,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/8ca6cb5a279b4dd6b7cedd97271ec683 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/8ca6cb5a279b4dd6b7cedd97271ec683 2024-12-06T14:42:12,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/47a81aa9298547ed99b9a0ba1d42b7c0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/47a81aa9298547ed99b9a0ba1d42b7c0 2024-12-06T14:42:12,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c43e138d9dc248d7baa6c195f932fbee to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c43e138d9dc248d7baa6c195f932fbee 2024-12-06T14:42:12,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/2d1883b5b1fc4823bbf2592aad7a1225 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/2d1883b5b1fc4823bbf2592aad7a1225 2024-12-06T14:42:12,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/03d744bc4e3547abb7df729217c0af8b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/03d744bc4e3547abb7df729217c0af8b 2024-12-06T14:42:12,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0276cad6b6004c2f9525206df31ac290 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0276cad6b6004c2f9525206df31ac290 2024-12-06T14:42:12,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/87bc282ecf1b4a26b1e86f6d359371fe to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/87bc282ecf1b4a26b1e86f6d359371fe 2024-12-06T14:42:12,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/204e6e7c9bb7456da8f5ce0b0301afe8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/204e6e7c9bb7456da8f5ce0b0301afe8 2024-12-06T14:42:12,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1963090d37ab460dbb910c949c376cf7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/1963090d37ab460dbb910c949c376cf7 2024-12-06T14:42:12,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/cce2b86022fc4a42bf27a292dd344fcc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/085032950ec641ef9d0dc1b28a319c56, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6440087aa24443f684f375716287a5f0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/c0d636cfb18e4231bd2e0d386d134eeb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/56afada2b8364ba493b4f9cf2046f903, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/048db0a6b4dd4ff693ed0b702ece4b9c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/a4ebafdfc68542d8afdbd66c5b585867, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/824108c8cef1442989448c1fb507b23b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d11cbce456ab4beea137825641d12aa3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/8e7051b48aaf47218f13986e4bc5219b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/0e436ea273234939b3f74f5d12da1a14, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d45cc7df40454703a3cb30d429cc0556, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ffea5d0ffc9d41aca7b4b66e4c856e30, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f4bddf081eae4f809ad4be7611bdc252, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e7a66e41f4474c508aed1d9236e6de3b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/acb44ab48c204dbba76c86adc7cefda2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7ed180ae92df4a60be2353d5050e8aa5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f299ad024d274fc78c58eb789a3a8569, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7e74ee65652543a8b39420b959aa245d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/840c036da67c4cf1aaf57c965946324b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ae1d2bef8a6f49be8e524548060ba4fb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/50196cc3d59c47ce901366af2229108c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6d2509e5edc64f0ebc4137d81f3d9b15, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/256bc4dceca74b39a4b8cf3f28ac8c13, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5185b62126f64c0cb88232dc3ba19a87, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f3db989e8d4a49388ebd7882599f749f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/30d7dda52aff4d65b1127dd80bbf1acc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c3cc1a6891546c2952cba68ba9f03fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5dc89e6fbb27413ea4416ce3e179b053, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/13deb4a325224761bc1c0271f0218942] to archive 2024-12-06T14:42:12,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:42:12,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/cce2b86022fc4a42bf27a292dd344fcc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/cce2b86022fc4a42bf27a292dd344fcc 2024-12-06T14:42:12,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/085032950ec641ef9d0dc1b28a319c56 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/085032950ec641ef9d0dc1b28a319c56 2024-12-06T14:42:12,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6440087aa24443f684f375716287a5f0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6440087aa24443f684f375716287a5f0 2024-12-06T14:42:12,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/c0d636cfb18e4231bd2e0d386d134eeb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/c0d636cfb18e4231bd2e0d386d134eeb 2024-12-06T14:42:12,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/56afada2b8364ba493b4f9cf2046f903 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/56afada2b8364ba493b4f9cf2046f903 2024-12-06T14:42:12,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/048db0a6b4dd4ff693ed0b702ece4b9c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/048db0a6b4dd4ff693ed0b702ece4b9c 2024-12-06T14:42:12,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/a4ebafdfc68542d8afdbd66c5b585867 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/a4ebafdfc68542d8afdbd66c5b585867 2024-12-06T14:42:12,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/824108c8cef1442989448c1fb507b23b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/824108c8cef1442989448c1fb507b23b 2024-12-06T14:42:12,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d11cbce456ab4beea137825641d12aa3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d11cbce456ab4beea137825641d12aa3 2024-12-06T14:42:12,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/8e7051b48aaf47218f13986e4bc5219b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/8e7051b48aaf47218f13986e4bc5219b 2024-12-06T14:42:12,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/0e436ea273234939b3f74f5d12da1a14 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/0e436ea273234939b3f74f5d12da1a14 2024-12-06T14:42:12,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e05b0a0b05de4e0e8e6ffbe08a27d0d1 2024-12-06T14:42:12,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d45cc7df40454703a3cb30d429cc0556 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/d45cc7df40454703a3cb30d429cc0556 2024-12-06T14:42:12,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ffea5d0ffc9d41aca7b4b66e4c856e30 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ffea5d0ffc9d41aca7b4b66e4c856e30 2024-12-06T14:42:12,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f4bddf081eae4f809ad4be7611bdc252 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f4bddf081eae4f809ad4be7611bdc252 2024-12-06T14:42:12,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e7a66e41f4474c508aed1d9236e6de3b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/e7a66e41f4474c508aed1d9236e6de3b 2024-12-06T14:42:12,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/acb44ab48c204dbba76c86adc7cefda2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/acb44ab48c204dbba76c86adc7cefda2 2024-12-06T14:42:12,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7ed180ae92df4a60be2353d5050e8aa5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7ed180ae92df4a60be2353d5050e8aa5 2024-12-06T14:42:12,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f299ad024d274fc78c58eb789a3a8569 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f299ad024d274fc78c58eb789a3a8569 2024-12-06T14:42:12,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7e74ee65652543a8b39420b959aa245d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/7e74ee65652543a8b39420b959aa245d 2024-12-06T14:42:12,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/840c036da67c4cf1aaf57c965946324b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/840c036da67c4cf1aaf57c965946324b 2024-12-06T14:42:12,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ae1d2bef8a6f49be8e524548060ba4fb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/ae1d2bef8a6f49be8e524548060ba4fb 2024-12-06T14:42:12,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/50196cc3d59c47ce901366af2229108c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/50196cc3d59c47ce901366af2229108c 2024-12-06T14:42:12,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6d2509e5edc64f0ebc4137d81f3d9b15 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6d2509e5edc64f0ebc4137d81f3d9b15 2024-12-06T14:42:12,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/256bc4dceca74b39a4b8cf3f28ac8c13 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/256bc4dceca74b39a4b8cf3f28ac8c13 2024-12-06T14:42:12,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5185b62126f64c0cb88232dc3ba19a87 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5185b62126f64c0cb88232dc3ba19a87 2024-12-06T14:42:12,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f3db989e8d4a49388ebd7882599f749f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f3db989e8d4a49388ebd7882599f749f 2024-12-06T14:42:12,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/30d7dda52aff4d65b1127dd80bbf1acc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/30d7dda52aff4d65b1127dd80bbf1acc 2024-12-06T14:42:12,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c3cc1a6891546c2952cba68ba9f03fa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c3cc1a6891546c2952cba68ba9f03fa 2024-12-06T14:42:12,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5dc89e6fbb27413ea4416ce3e179b053 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5dc89e6fbb27413ea4416ce3e179b053 2024-12-06T14:42:12,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/13deb4a325224761bc1c0271f0218942 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/13deb4a325224761bc1c0271f0218942 2024-12-06T14:42:12,275 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/recovered.edits/528.seqid, newMaxSeqId=528, maxSeqId=1 2024-12-06T14:42:12,276 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. 2024-12-06T14:42:12,276 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 709a255a23a9b4b1363f2ec84dd61843: Waiting for close lock at 1733496130876Running coprocessor pre-close hooks at 1733496130876Disabling compacts and flushes for region at 1733496130877 (+1 ms)Disabling writes for close at 1733496130877Obtaining lock to block concurrent updates at 1733496130877Preparing flush snapshotting stores in 709a255a23a9b4b1363f2ec84dd61843 at 1733496130877Finished memstore snapshotting TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733496130879 (+2 ms)Flushing stores of TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843. at 1733496130880 (+1 ms)Flushing 709a255a23a9b4b1363f2ec84dd61843/A: creating writer at 1733496130880Flushing 709a255a23a9b4b1363f2ec84dd61843/A: appending metadata at 1733496130885 (+5 ms)Flushing 709a255a23a9b4b1363f2ec84dd61843/A: closing flushed file at 1733496130885Flushing 709a255a23a9b4b1363f2ec84dd61843/B: creating writer at 1733496131294 (+409 ms)Flushing 709a255a23a9b4b1363f2ec84dd61843/B: appending metadata at 1733496131297 (+3 ms)Flushing 709a255a23a9b4b1363f2ec84dd61843/B: closing flushed file at 1733496131297Flushing 709a255a23a9b4b1363f2ec84dd61843/C: creating writer at 1733496131708 (+411 ms)Flushing 709a255a23a9b4b1363f2ec84dd61843/C: appending metadata at 1733496131712 (+4 ms)Flushing 709a255a23a9b4b1363f2ec84dd61843/C: closing flushed file at 1733496131712Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4deba973: reopening flushed file at 1733496132124 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51d79cb8: reopening flushed file at 1733496132130 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50b93763: reopening flushed file at 1733496132136 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 709a255a23a9b4b1363f2ec84dd61843 in 1265ms, sequenceid=525, compaction requested=true at 1733496132142 (+6 ms)Writing region close event to WAL at 1733496132272 (+130 ms)Running coprocessor post-close hooks at 1733496132276 (+4 ms)Closed at 1733496132276 2024-12-06T14:42:12,278 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:12,279 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=709a255a23a9b4b1363f2ec84dd61843, regionState=CLOSED 2024-12-06T14:42:12,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:42:12,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-06T14:42:12,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 709a255a23a9b4b1363f2ec84dd61843, server=bb7ea3e46d9c,35831,1733496100253 in 1.5660 sec 2024-12-06T14:42:12,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-06T14:42:12,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=709a255a23a9b4b1363f2ec84dd61843, UNASSIGN in 1.5730 sec 2024-12-06T14:42:12,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-06T14:42:12,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5790 sec 2024-12-06T14:42:12,287 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496132287"}]},"ts":"1733496132287"} 2024-12-06T14:42:12,289 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T14:42:12,289 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T14:42:12,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-12-06T14:42:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T14:42:12,861 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:12,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T14:42:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:12,873 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T14:42:12,875 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:12,879 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:12,884 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/recovered.edits] 2024-12-06T14:42:12,888 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/b193ab5b943f483485bae48c8a481ada to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/b193ab5b943f483485bae48c8a481ada 2024-12-06T14:42:12,889 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c7220d0c846449e5a3be595dd0679da7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/c7220d0c846449e5a3be595dd0679da7 2024-12-06T14:42:12,891 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/feace0a9aa1f473889a5e16aece49ec0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/A/feace0a9aa1f473889a5e16aece49ec0 2024-12-06T14:42:12,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0ff1fd8b23dd444487db6a30b11d5168 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/0ff1fd8b23dd444487db6a30b11d5168 2024-12-06T14:42:12,895 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/9e535ebf0ba64ed7881cffc6bb827482 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/9e535ebf0ba64ed7881cffc6bb827482 2024-12-06T14:42:12,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0ab52a2e9c647a8a0e6fc2fb40ac1e9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/B/c0ab52a2e9c647a8a0e6fc2fb40ac1e9 2024-12-06T14:42:12,899 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c0251fdf71a48ae9eb4030cb648ff0d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/5c0251fdf71a48ae9eb4030cb648ff0d 2024-12-06T14:42:12,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6ba5dd74eab24c8087384d6766dde731 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/6ba5dd74eab24c8087384d6766dde731 2024-12-06T14:42:12,901 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f456a9915b59479c921be4cfef3fdaa7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/C/f456a9915b59479c921be4cfef3fdaa7 2024-12-06T14:42:12,905 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/recovered.edits/528.seqid to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843/recovered.edits/528.seqid 2024-12-06T14:42:12,905 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/709a255a23a9b4b1363f2ec84dd61843 2024-12-06T14:42:12,905 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T14:42:12,909 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:12,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-06T14:42:12,918 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T14:42:12,944 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T14:42:12,946 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:12,946 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T14:42:12,946 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733496132946"}]},"ts":"9223372036854775807"} 2024-12-06T14:42:12,950 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T14:42:12,950 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 709a255a23a9b4b1363f2ec84dd61843, NAME => 'TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T14:42:12,951 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T14:42:12,951 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733496132951"}]},"ts":"9223372036854775807"} 2024-12-06T14:42:12,953 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T14:42:12,954 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:12,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-12-06T14:42:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T14:42:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T14:42:12,979 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:12,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_912585382_22 at /127.0.0.1:33282 [Waiting for operation #309] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;bb7ea3e46d9c:35831-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1216782701_22 at /127.0.0.1:50144 [Waiting for operation #320] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 187) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5719 (was 6343) 2024-12-06T14:42:12,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=5718 2024-12-06T14:42:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:42:13,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:42:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:13,002 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:42:13,002 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:13,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-06T14:42:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-06T14:42:13,004 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:42:13,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741939_1115 (size=1061) 2024-12-06T14:42:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-06T14:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-06T14:42:13,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:42:13,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741940_1116 (size=53) 2024-12-06T14:42:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-06T14:42:13,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:42:13,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c5dbf7aedc4a0da1b71009ce70eea907, disabling compactions & flushes 2024-12-06T14:42:13,826 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:13,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:13,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. after waiting 0 ms 2024-12-06T14:42:13,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:13,826 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:13,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c5dbf7aedc4a0da1b71009ce70eea907: Waiting for close lock at 1733496133826Disabling compacts and flushes for region at 1733496133826Disabling writes for close at 1733496133826Writing region close event to WAL at 1733496133826Closed at 1733496133826 2024-12-06T14:42:13,828 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:42:13,829 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733496133828"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496133828"}]},"ts":"1733496133828"} 2024-12-06T14:42:13,831 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:42:13,832 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:42:13,832 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496133832"}]},"ts":"1733496133832"} 2024-12-06T14:42:13,834 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T14:42:13,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, ASSIGN}] 2024-12-06T14:42:13,836 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, ASSIGN 2024-12-06T14:42:13,837 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=false 2024-12-06T14:42:13,988 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:13,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, ASSIGN because future has completed 2024-12-06T14:42:13,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-06T14:42:14,152 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:14,153 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:42:14,153 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,153 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:42:14,153 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,154 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,156 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,157 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:14,158 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5dbf7aedc4a0da1b71009ce70eea907 columnFamilyName A 2024-12-06T14:42:14,158 DEBUG [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:14,158 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(327): Store=c5dbf7aedc4a0da1b71009ce70eea907/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:14,158 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,159 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:14,160 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5dbf7aedc4a0da1b71009ce70eea907 columnFamilyName B 2024-12-06T14:42:14,160 DEBUG [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:14,160 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(327): Store=c5dbf7aedc4a0da1b71009ce70eea907/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:14,160 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,161 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:14,161 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5dbf7aedc4a0da1b71009ce70eea907 columnFamilyName C 2024-12-06T14:42:14,162 DEBUG [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:14,162 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(327): Store=c5dbf7aedc4a0da1b71009ce70eea907/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:14,162 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:14,162 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,163 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,163 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,164 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,164 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,165 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:42:14,166 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,167 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:42:14,168 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c5dbf7aedc4a0da1b71009ce70eea907; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72173689, jitterRate=0.07547177374362946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:42:14,168 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:14,169 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c5dbf7aedc4a0da1b71009ce70eea907: Running coprocessor pre-open hook at 1733496134154Writing region info on filesystem at 1733496134154Initializing all the Stores at 1733496134155 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496134155Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496134155Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496134155Cleaning up temporary data from old regions at 1733496134164 (+9 ms)Running coprocessor post-open hooks at 1733496134168 (+4 ms)Region opened successfully at 1733496134168 2024-12-06T14:42:14,169 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., pid=32, masterSystemTime=1733496134145 2024-12-06T14:42:14,171 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:14,171 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:14,172 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:14,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:42:14,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-06T14:42:14,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 in 182 msec 2024-12-06T14:42:14,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-06T14:42:14,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, ASSIGN in 341 msec 2024-12-06T14:42:14,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:42:14,179 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496134179"}]},"ts":"1733496134179"} 2024-12-06T14:42:14,180 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T14:42:14,181 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:42:14,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-06T14:42:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-06T14:42:15,151 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:15,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:15,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:15,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:15,158 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c13cb7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:15,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccf8029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:15,162 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:15,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:15,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:42:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:42:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@227cf768 2024-12-06T14:42:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:42:15,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:42:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:42:15,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:42:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:15,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741941_1117 (size=1097) 2024-12-06T14:42:15,671 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-06T14:42:15,671 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-06T14:42:15,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:42:15,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, REOPEN/MOVE}] 2024-12-06T14:42:15,686 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, REOPEN/MOVE 2024-12-06T14:42:15,687 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:15,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, REOPEN/MOVE because future has completed 2024-12-06T14:42:15,689 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:42:15,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:42:15,843 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:15,843 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:42:15,843 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing c5dbf7aedc4a0da1b71009ce70eea907, disabling compactions & flushes 2024-12-06T14:42:15,843 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:15,843 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:15,843 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. after waiting 0 ms 2024-12-06T14:42:15,843 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:15,850 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T14:42:15,851 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:15,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for c5dbf7aedc4a0da1b71009ce70eea907: Waiting for close lock at 1733496135843Running coprocessor pre-close hooks at 1733496135843Disabling compacts and flushes for region at 1733496135843Disabling writes for close at 1733496135843Writing region close event to WAL at 1733496135844 (+1 ms)Running coprocessor post-close hooks at 1733496135851 (+7 ms)Closed at 1733496135851 2024-12-06T14:42:15,851 WARN [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: c5dbf7aedc4a0da1b71009ce70eea907 to self. 2024-12-06T14:42:15,854 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:15,855 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=CLOSED 2024-12-06T14:42:15,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:42:15,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-06T14:42:15,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 in 170 msec 2024-12-06T14:42:15,862 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, REOPEN/MOVE; state=CLOSED, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=true 2024-12-06T14:42:16,013 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, REOPEN/MOVE because future has completed 2024-12-06T14:42:16,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:42:16,177 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,178 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:42:16,178 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,178 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:42:16,178 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,179 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,184 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,185 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:16,190 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5dbf7aedc4a0da1b71009ce70eea907 columnFamilyName A 2024-12-06T14:42:16,192 DEBUG [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:16,193 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(327): Store=c5dbf7aedc4a0da1b71009ce70eea907/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:16,193 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,194 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:16,194 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5dbf7aedc4a0da1b71009ce70eea907 columnFamilyName B 2024-12-06T14:42:16,194 DEBUG [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:16,194 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(327): Store=c5dbf7aedc4a0da1b71009ce70eea907/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:16,194 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,195 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:16,195 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5dbf7aedc4a0da1b71009ce70eea907 columnFamilyName C 2024-12-06T14:42:16,195 DEBUG [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:16,196 INFO [StoreOpener-c5dbf7aedc4a0da1b71009ce70eea907-1 {}] regionserver.HStore(327): Store=c5dbf7aedc4a0da1b71009ce70eea907/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:16,196 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,196 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,196 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,198 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,199 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,199 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,199 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:42:16,200 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,201 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c5dbf7aedc4a0da1b71009ce70eea907; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66883420, jitterRate=-0.003359377384185791}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:42:16,201 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,203 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c5dbf7aedc4a0da1b71009ce70eea907: Running coprocessor pre-open hook at 1733496136179Writing region info on filesystem at 1733496136179Initializing all the Stores at 1733496136180 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496136180Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496136180Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496136180Cleaning up temporary data from old regions at 1733496136199 (+19 ms)Running coprocessor post-open hooks at 1733496136201 (+2 ms)Region opened successfully at 1733496136203 (+2 ms) 2024-12-06T14:42:16,204 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., pid=37, masterSystemTime=1733496136172 2024-12-06T14:42:16,206 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,206 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,207 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=OPEN, openSeqNum=5, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:42:16,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-06T14:42:16,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 in 191 msec 2024-12-06T14:42:16,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-06T14:42:16,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, REOPEN/MOVE in 526 msec 2024-12-06T14:42:16,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-06T14:42:16,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-12-06T14:42:16,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0400 sec 2024-12-06T14:42:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-06T14:42:16,220 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:16,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,222 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a91a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b42a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,225 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,227 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d374d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286d9d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea2e03d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,233 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cef8452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,234 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c75956e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,236 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e96db42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,237 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ade830f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,239 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5327b288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234ac34c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c562e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,243 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e4063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,244 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdf07c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c7e2da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,248 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22644b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fb429c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:16,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:16,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784dfc97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:16,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:16,252 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,253 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,253 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,253 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,253 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:16,253 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,254 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,254 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,254 DEBUG [Thread-615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-06T14:42:16,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,256 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:16,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:16,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T14:42:16,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:16,259 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:16,260 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:16,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:42:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:42:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:16,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120613cddeb265114ea689c556ed0d9dc762_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496136268/Put/seqid=0 2024-12-06T14:42:16,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496136323, max timestamp=1733496136334Num uniques 859; 2024-12-06T14:42:16,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:16,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496136323, max timestamp=1733496136334Num uniques 859; 2024-12-06T14:42:16,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496136323, max timestamp=1733496136334Num uniques 859; 2024-12-06T14:42:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:16,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741942_1118 (size=12154) 2024-12-06T14:42:16,361 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:16,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120613cddeb265114ea689c556ed0d9dc762_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120613cddeb265114ea689c556ed0d9dc762_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:16,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fee18ded821e4780bc69ee336b087a77, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T14:42:16,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fee18ded821e4780bc69ee336b087a77 is 175, key is test_row_0/A:col10/1733496136268/Put/seqid=0 2024-12-06T14:42:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496196373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496196373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496196376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496196378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496196379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741943_1119 (size=30955) 2024-12-06T14:42:16,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fee18ded821e4780bc69ee336b087a77 2024-12-06T14:42:16,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/e0aef5e7c048434eb78024aa7b24a948 is 50, key is test_row_0/B:col10/1733496136268/Put/seqid=0 2024-12-06T14:42:16,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741944_1120 (size=12001) 2024-12-06T14:42:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496196489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496196491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496196499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496196499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496196499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T14:42:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496196699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496196699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496196708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496196708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496196709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:16,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/e0aef5e7c048434eb78024aa7b24a948 2024-12-06T14:42:16,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/5b448b3a66b34db69ce05ebf53e5c59b is 50, key is test_row_0/C:col10/1733496136268/Put/seqid=0 2024-12-06T14:42:16,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:16,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:16,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741945_1121 (size=12001) 2024-12-06T14:42:16,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/5b448b3a66b34db69ce05ebf53e5c59b 2024-12-06T14:42:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T14:42:16,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fee18ded821e4780bc69ee336b087a77 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77 2024-12-06T14:42:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77, entries=150, sequenceid=15, filesize=30.2 K 2024-12-06T14:42:16,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/e0aef5e7c048434eb78024aa7b24a948 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/e0aef5e7c048434eb78024aa7b24a948 2024-12-06T14:42:16,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/e0aef5e7c048434eb78024aa7b24a948, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T14:42:16,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/5b448b3a66b34db69ce05ebf53e5c59b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/5b448b3a66b34db69ce05ebf53e5c59b 2024-12-06T14:42:16,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/5b448b3a66b34db69ce05ebf53e5c59b, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T14:42:16,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c5dbf7aedc4a0da1b71009ce70eea907 in 663ms, sequenceid=15, compaction requested=false 2024-12-06T14:42:16,936 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T14:42:16,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:17,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496136370, max timestamp=1733496137009Num uniques 1018; 2024-12-06T14:42:17,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:42:17,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496136375, max timestamp=1733496137009Num uniques 1018; 2024-12-06T14:42:17,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-06T14:42:17,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496136375, max timestamp=1733496137009Num uniques 1018; 2024-12-06T14:42:17,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-06T14:42:17,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=40.89 KB heapSize=106.66 KB 2024-12-06T14:42:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:17,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064d9935a668b24600895bfed0062046ef_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496137009/Put/seqid=0 2024-12-06T14:42:17,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496137043, max timestamp=1733496137058Num uniques 859; 2024-12-06T14:42:17,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:17,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496137043, max timestamp=1733496137058Num uniques 859; 2024-12-06T14:42:17,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496137043, max timestamp=1733496137058Num uniques 859; 2024-12-06T14:42:17,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:17,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:17,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741946_1122 (size=17034) 2024-12-06T14:42:17,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:17,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064d9935a668b24600895bfed0062046ef_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064d9935a668b24600895bfed0062046ef_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:17,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/cf118bd5f78c49778a6f0bed23c46ea9, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:17,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/cf118bd5f78c49778a6f0bed23c46ea9 is 175, key is test_row_0/A:col10/1733496137009/Put/seqid=0 2024-12-06T14:42:17,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741947_1123 (size=48139) 2024-12-06T14:42:17,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/cf118bd5f78c49778a6f0bed23c46ea9 2024-12-06T14:42:17,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733496137080, max timestamp=1733496137096Num uniques 1018; 2024-12-06T14:42:17,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-06T14:42:17,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733496137080, max timestamp=1733496137096Num uniques 1018; 2024-12-06T14:42:17,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-06T14:42:17,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733496137080, max timestamp=1733496137096Num uniques 1018; 2024-12-06T14:42:17,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-06T14:42:17,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/59f2859c6a5d43f9b676d5b22583efa9 is 50, key is test_row_0/B:col10/1733496137009/Put/seqid=0 2024-12-06T14:42:17,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741948_1124 (size=12001) 2024-12-06T14:42:17,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1733496137118, max timestamp=1733496137132Num uniques 1027; 2024-12-06T14:42:17,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1733496137118, max timestamp=1733496137132Num uniques 1027; 2024-12-06T14:42:17,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-12-06T14:42:17,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-12-06T14:42:17,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1733496137118, max timestamp=1733496137132Num uniques 1027; 2024-12-06T14:42:17,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-12-06T14:42:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496197153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496197153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496197157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496197158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496197159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496197268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496197268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496197268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496197279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496197282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T14:42:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496197478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496197478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496197478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496197489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496197489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.62 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/59f2859c6a5d43f9b676d5b22583efa9 2024-12-06T14:42:17,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b670de1a25604e1ba11db2d287ad46b8 is 50, key is test_row_0/C:col10/1733496137009/Put/seqid=0 2024-12-06T14:42:17,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741949_1125 (size=12001) 2024-12-06T14:42:17,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496197788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496197788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496197788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496197799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496197799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:17,806 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:42:17,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.62 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b670de1a25604e1ba11db2d287ad46b8 2024-12-06T14:42:17,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/cf118bd5f78c49778a6f0bed23c46ea9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9 2024-12-06T14:42:17,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:17,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:17,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9, entries=250, sequenceid=55, filesize=47.0 K 2024-12-06T14:42:17,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/59f2859c6a5d43f9b676d5b22583efa9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/59f2859c6a5d43f9b676d5b22583efa9 2024-12-06T14:42:17,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/59f2859c6a5d43f9b676d5b22583efa9, entries=150, sequenceid=55, filesize=11.7 K 2024-12-06T14:42:17,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b670de1a25604e1ba11db2d287ad46b8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b670de1a25604e1ba11db2d287ad46b8 2024-12-06T14:42:17,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b670de1a25604e1ba11db2d287ad46b8, entries=150, sequenceid=55, filesize=11.7 K 2024-12-06T14:42:17,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.89 KB/41874, heapSize ~106.61 KB/109168, currentSize=154.31 KB/158010 for c5dbf7aedc4a0da1b71009ce70eea907 in 950ms, sequenceid=55, compaction requested=false 2024-12-06T14:42:17,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:18,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T14:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,106 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=154.31 KB heapSize=424.31 KB 2024-12-06T14:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:18,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e5ad0504a0424403afe10e8d641854e1_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496137155/Put/seqid=0 2024-12-06T14:42:18,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741950_1126 (size=12154) 2024-12-06T14:42:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,133 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e5ad0504a0424403afe10e8d641854e1_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e5ad0504a0424403afe10e8d641854e1_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/940f561fa4384c8993397f5637a680dd, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:18,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/940f561fa4384c8993397f5637a680dd is 175, key is test_row_0/A:col10/1733496137155/Put/seqid=0 2024-12-06T14:42:18,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741951_1127 (size=30955) 2024-12-06T14:42:18,161 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/940f561fa4384c8993397f5637a680dd 2024-12-06T14:42:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/c6864abaf7d04da09ea9948f3854d50a is 50, key is test_row_0/B:col10/1733496137155/Put/seqid=0 2024-12-06T14:42:18,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741952_1128 (size=12001) 2024-12-06T14:42:18,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/c6864abaf7d04da09ea9948f3854d50a 2024-12-06T14:42:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b743583cc3e1495182d058a185b0bde1 is 50, key is test_row_0/C:col10/1733496137155/Put/seqid=0 2024-12-06T14:42:18,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741953_1129 (size=12001) 2024-12-06T14:42:18,227 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b743583cc3e1495182d058a185b0bde1 2024-12-06T14:42:18,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/940f561fa4384c8993397f5637a680dd as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd 2024-12-06T14:42:18,243 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd, entries=150, sequenceid=125, filesize=30.2 K 2024-12-06T14:42:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/c6864abaf7d04da09ea9948f3854d50a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c6864abaf7d04da09ea9948f3854d50a 2024-12-06T14:42:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,253 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c6864abaf7d04da09ea9948f3854d50a, entries=150, sequenceid=125, filesize=11.7 K 2024-12-06T14:42:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b743583cc3e1495182d058a185b0bde1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b743583cc3e1495182d058a185b0bde1 2024-12-06T14:42:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,261 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b743583cc3e1495182d058a185b0bde1, entries=150, sequenceid=125, filesize=11.7 K 2024-12-06T14:42:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,262 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~424.27 KB/434448, currentSize=0 B/0 for c5dbf7aedc4a0da1b71009ce70eea907 in 156ms, sequenceid=125, compaction requested=true 2024-12-06T14:42:18,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:18,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-06T14:42:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-06T14:42:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-06T14:42:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0030 sec 2024-12-06T14:42:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.0160 sec 2024-12-06T14:42:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:42:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:18,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:18,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b558c5c1b6f645ec8560d3d9dfe6f80c_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T14:42:18,408 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:18,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-06T14:42:18,410 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:18,411 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:18,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:18,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741955_1131 (size=24758) 2024-12-06T14:42:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733496138411, max timestamp=1733496138428Num uniques 859; 2024-12-06T14:42:18,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733496138411, max timestamp=1733496138428Num uniques 859; 2024-12-06T14:42:18,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733496138411, max timestamp=1733496138428Num uniques 859; 2024-12-06T14:42:18,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T14:42:18,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T14:42:18,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T14:42:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496198426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496198445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496198447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496198448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496198449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496198538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496198558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496198558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496198559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496198559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:18,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496198748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496198768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496198769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496198769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496198769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:18,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:18,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b558c5c1b6f645ec8560d3d9dfe6f80c_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b558c5c1b6f645ec8560d3d9dfe6f80c_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:18,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0866e13f759b46229e1099501de4f82c, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:18,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0866e13f759b46229e1099501de4f82c is 175, key is test_row_0/A:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:18,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741954_1130 (size=74395) 2024-12-06T14:42:18,839 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0866e13f759b46229e1099501de4f82c 2024-12-06T14:42:18,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/91a1a76ae91a4deb8c485abf12229705 is 50, key is test_row_0/B:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:18,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741956_1132 (size=12151) 2024-12-06T14:42:18,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:18,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:19,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496199058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496199079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496199088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496199088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496199088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/91a1a76ae91a4deb8c485abf12229705 2024-12-06T14:42:19,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a3b9dbcd423447a2bd50c1b10c45cdc6 is 50, key is test_row_0/C:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:19,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741957_1133 (size=12151) 2024-12-06T14:42:19,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:19,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496199578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496199588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496199598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496199598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496199598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:19,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:19,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:19,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a3b9dbcd423447a2bd50c1b10c45cdc6 2024-12-06T14:42:19,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0866e13f759b46229e1099501de4f82c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c 2024-12-06T14:42:19,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c, entries=400, sequenceid=136, filesize=72.7 K 2024-12-06T14:42:19,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/91a1a76ae91a4deb8c485abf12229705 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/91a1a76ae91a4deb8c485abf12229705 2024-12-06T14:42:19,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/91a1a76ae91a4deb8c485abf12229705, entries=150, sequenceid=136, filesize=11.9 K 2024-12-06T14:42:19,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a3b9dbcd423447a2bd50c1b10c45cdc6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a3b9dbcd423447a2bd50c1b10c45cdc6 2024-12-06T14:42:19,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a3b9dbcd423447a2bd50c1b10c45cdc6, entries=150, sequenceid=136, filesize=11.9 K 2024-12-06T14:42:19,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c5dbf7aedc4a0da1b71009ce70eea907 in 1325ms, sequenceid=136, compaction requested=true 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:19,709 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:19,709 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:19,711 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 184444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:19,711 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:19,711 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/A is initiating minor compaction (all files) 2024-12-06T14:42:19,711 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/B is initiating minor compaction (all files) 2024-12-06T14:42:19,711 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/A in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,711 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/B in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,711 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/e0aef5e7c048434eb78024aa7b24a948, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/59f2859c6a5d43f9b676d5b22583efa9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c6864abaf7d04da09ea9948f3854d50a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/91a1a76ae91a4deb8c485abf12229705] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=47.0 K 2024-12-06T14:42:19,711 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=180.1 K 2024-12-06T14:42:19,711 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,711 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c] 2024-12-06T14:42:19,713 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e0aef5e7c048434eb78024aa7b24a948, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733496136268 2024-12-06T14:42:19,713 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting fee18ded821e4780bc69ee336b087a77, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733496136268 2024-12-06T14:42:19,713 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f2859c6a5d43f9b676d5b22583efa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733496137009 2024-12-06T14:42:19,714 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf118bd5f78c49778a6f0bed23c46ea9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733496136375 2024-12-06T14:42:19,714 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c6864abaf7d04da09ea9948f3854d50a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733496137152 2024-12-06T14:42:19,714 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940f561fa4384c8993397f5637a680dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733496137152 2024-12-06T14:42:19,714 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a1a76ae91a4deb8c485abf12229705, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733496138372 2024-12-06T14:42:19,715 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0866e13f759b46229e1099501de4f82c, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733496138369 2024-12-06T14:42:19,727 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:19,728 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f3b8a5a9f3c346e38c3bbb19e8e54c64 is 50, key is test_row_0/B:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:19,730 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:19,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741958_1134 (size=12289) 2024-12-06T14:42:19,735 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206f1b2649e4fdd44bba90054955925c4e6_c5dbf7aedc4a0da1b71009ce70eea907 store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:19,740 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206f1b2649e4fdd44bba90054955925c4e6_c5dbf7aedc4a0da1b71009ce70eea907, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:19,740 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f1b2649e4fdd44bba90054955925c4e6_c5dbf7aedc4a0da1b71009ce70eea907 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:19,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741959_1135 (size=4469) 2024-12-06T14:42:19,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T14:42:19,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:19,812 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=134.18 KB heapSize=361.55 KB 2024-12-06T14:42:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:19,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ba8891f1525446ce8579164d84e439a4_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496138446/Put/seqid=0 2024-12-06T14:42:19,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741960_1136 (size=12304) 2024-12-06T14:42:20,140 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f3b8a5a9f3c346e38c3bbb19e8e54c64 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f3b8a5a9f3c346e38c3bbb19e8e54c64 2024-12-06T14:42:20,145 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/B of c5dbf7aedc4a0da1b71009ce70eea907 into f3b8a5a9f3c346e38c3bbb19e8e54c64(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:20,145 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:20,145 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/B, priority=12, startTime=1733496139709; duration=0sec 2024-12-06T14:42:20,145 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:20,145 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:B 2024-12-06T14:42:20,145 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:20,147 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:20,147 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/C is initiating minor compaction (all files) 2024-12-06T14:42:20,147 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/C in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:20,147 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/5b448b3a66b34db69ce05ebf53e5c59b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b670de1a25604e1ba11db2d287ad46b8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b743583cc3e1495182d058a185b0bde1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a3b9dbcd423447a2bd50c1b10c45cdc6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=47.0 K 2024-12-06T14:42:20,147 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b448b3a66b34db69ce05ebf53e5c59b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733496136268 2024-12-06T14:42:20,147 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b670de1a25604e1ba11db2d287ad46b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733496137009 2024-12-06T14:42:20,148 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b743583cc3e1495182d058a185b0bde1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733496137152 2024-12-06T14:42:20,148 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b9dbcd423447a2bd50c1b10c45cdc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733496138372 2024-12-06T14:42:20,155 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:20,156 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/c663f0f33d3b4d60aa5d96ba989ee69e is 50, key is test_row_0/C:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:20,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741961_1137 (size=12289) 2024-12-06T14:42:20,179 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#A#compaction#115 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:20,180 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/708e557cba22477db9a180685c948038 is 175, key is test_row_0/A:col10/1733496138384/Put/seqid=0 2024-12-06T14:42:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:20,246 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ba8891f1525446ce8579164d84e439a4_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ba8891f1525446ce8579164d84e439a4_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:20,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0378669dba084bf687f03805ecd0ca42, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:20,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0378669dba084bf687f03805ecd0ca42 is 175, key is test_row_0/A:col10/1733496138446/Put/seqid=0 2024-12-06T14:42:20,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741962_1138 (size=31243) 2024-12-06T14:42:20,274 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/708e557cba22477db9a180685c948038 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/708e557cba22477db9a180685c948038 2024-12-06T14:42:20,287 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/A of c5dbf7aedc4a0da1b71009ce70eea907 into 708e557cba22477db9a180685c948038(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:20,287 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:20,287 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/A, priority=12, startTime=1733496139709; duration=0sec 2024-12-06T14:42:20,287 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:20,287 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:A 2024-12-06T14:42:20,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741963_1139 (size=31105) 2024-12-06T14:42:20,290 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0378669dba084bf687f03805ecd0ca42 2024-12-06T14:42:20,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/8220bd2ca0f14890b282e694e2af49a7 is 50, key is test_row_0/B:col10/1733496138446/Put/seqid=0 2024-12-06T14:42:20,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741964_1140 (size=12151) 2024-12-06T14:42:20,343 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/8220bd2ca0f14890b282e694e2af49a7 2024-12-06T14:42:20,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a06d44680cfd4b7b841b79fadf5064a8 is 50, key is test_row_0/C:col10/1733496138446/Put/seqid=0 2024-12-06T14:42:20,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741965_1141 (size=12151) 2024-12-06T14:42:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:20,578 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/c663f0f33d3b4d60aa5d96ba989ee69e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c663f0f33d3b4d60aa5d96ba989ee69e 2024-12-06T14:42:20,588 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/C of c5dbf7aedc4a0da1b71009ce70eea907 into c663f0f33d3b4d60aa5d96ba989ee69e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:20,589 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:20,589 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/C, priority=12, startTime=1733496139709; duration=0sec 2024-12-06T14:42:20,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:20,592 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:20,592 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:C 2024-12-06T14:42:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496200613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496200613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496200615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496200615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496200618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,625 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496200729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496200729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496200729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496200729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,787 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a06d44680cfd4b7b841b79fadf5064a8 2024-12-06T14:42:20,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0378669dba084bf687f03805ecd0ca42 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42 2024-12-06T14:42:20,799 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42, entries=150, sequenceid=168, filesize=30.4 K 2024-12-06T14:42:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/8220bd2ca0f14890b282e694e2af49a7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8220bd2ca0f14890b282e694e2af49a7 2024-12-06T14:42:20,807 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8220bd2ca0f14890b282e694e2af49a7, entries=150, sequenceid=168, filesize=11.9 K 2024-12-06T14:42:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a06d44680cfd4b7b841b79fadf5064a8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a06d44680cfd4b7b841b79fadf5064a8 2024-12-06T14:42:20,815 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a06d44680cfd4b7b841b79fadf5064a8, entries=150, sequenceid=168, filesize=11.9 K 2024-12-06T14:42:20,816 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~361.50 KB/370176, currentSize=60.38 KB/61830 for c5dbf7aedc4a0da1b71009ce70eea907 in 1005ms, sequenceid=168, compaction requested=false 2024-12-06T14:42:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-06T14:42:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-06T14:42:20,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-06T14:42:20,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4060 sec 2024-12-06T14:42:20,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.4120 sec 2024-12-06T14:42:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:20,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T14:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:20,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206289b04db511b456ba6d74ecc354e6828_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496200963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496200966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496200966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496200970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:20,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741966_1142 (size=12304) 2024-12-06T14:42:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496201078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496201079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496201079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496201079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496201288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496201289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496201289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496201289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,383 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:21,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206289b04db511b456ba6d74ecc354e6828_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206289b04db511b456ba6d74ecc354e6828_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:21,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/09e5557af648407c9a030fbb6446878b, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:21,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/09e5557af648407c9a030fbb6446878b is 175, key is test_row_0/A:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:21,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741967_1143 (size=31105) 2024-12-06T14:42:21,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496201598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496201598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496201598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496201598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:21,795 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/09e5557af648407c9a030fbb6446878b 2024-12-06T14:42:21,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/82f2eed446f2498aadea580effd99632 is 50, key is test_row_0/B:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:21,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741968_1144 (size=12151) 2024-12-06T14:42:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496202108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496202108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:22,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496202108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:22,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496202108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:22,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/82f2eed446f2498aadea580effd99632 2024-12-06T14:42:22,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/aa85d3b5f4b642f2a495134a802b4f1a is 50, key is test_row_0/C:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741969_1145 (size=12151) 2024-12-06T14:42:22,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/aa85d3b5f4b642f2a495134a802b4f1a 2024-12-06T14:42:22,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/09e5557af648407c9a030fbb6446878b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b 2024-12-06T14:42:22,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b, entries=150, sequenceid=185, filesize=30.4 K 2024-12-06T14:42:22,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/82f2eed446f2498aadea580effd99632 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/82f2eed446f2498aadea580effd99632 2024-12-06T14:42:22,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/82f2eed446f2498aadea580effd99632, entries=150, sequenceid=185, filesize=11.9 K 2024-12-06T14:42:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/aa85d3b5f4b642f2a495134a802b4f1a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/aa85d3b5f4b642f2a495134a802b4f1a 2024-12-06T14:42:22,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/aa85d3b5f4b642f2a495134a802b4f1a, entries=150, sequenceid=185, filesize=11.9 K 2024-12-06T14:42:22,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c5dbf7aedc4a0da1b71009ce70eea907 in 1321ms, sequenceid=185, compaction requested=true 2024-12-06T14:42:22,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:22,262 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:22,262 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:22,263 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:22,263 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:22,263 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/B is initiating minor compaction (all files) 2024-12-06T14:42:22,263 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/A is initiating minor compaction (all files) 2024-12-06T14:42:22,263 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/B in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,263 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/A in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,263 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f3b8a5a9f3c346e38c3bbb19e8e54c64, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8220bd2ca0f14890b282e694e2af49a7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/82f2eed446f2498aadea580effd99632] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=35.7 K 2024-12-06T14:42:22,263 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/708e557cba22477db9a180685c948038, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=91.3 K 2024-12-06T14:42:22,263 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,263 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/708e557cba22477db9a180685c948038, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b] 2024-12-06T14:42:22,264 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b8a5a9f3c346e38c3bbb19e8e54c64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733496138372 2024-12-06T14:42:22,264 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708e557cba22477db9a180685c948038, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733496138372 2024-12-06T14:42:22,264 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0378669dba084bf687f03805ecd0ca42, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733496138428 2024-12-06T14:42:22,264 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8220bd2ca0f14890b282e694e2af49a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733496138428 2024-12-06T14:42:22,264 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f2eed446f2498aadea580effd99632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733496140613 2024-12-06T14:42:22,264 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e5557af648407c9a030fbb6446878b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733496140613 2024-12-06T14:42:22,280 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:22,281 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:22,281 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6d62711d02e749518a2b49c92a6bdd4b is 50, key is test_row_0/B:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:22,285 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206a0a079ed67eb446b8559eba8240e551e_c5dbf7aedc4a0da1b71009ce70eea907 store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:22,297 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206a0a079ed67eb446b8559eba8240e551e_c5dbf7aedc4a0da1b71009ce70eea907, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:22,297 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a0a079ed67eb446b8559eba8240e551e_c5dbf7aedc4a0da1b71009ce70eea907 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:22,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741970_1146 (size=12391) 2024-12-06T14:42:22,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741971_1147 (size=4469) 2024-12-06T14:42:22,308 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#A#compaction#124 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:22,309 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1 is 175, key is test_row_0/A:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:22,314 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6d62711d02e749518a2b49c92a6bdd4b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6d62711d02e749518a2b49c92a6bdd4b 2024-12-06T14:42:22,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741972_1148 (size=31345) 2024-12-06T14:42:22,323 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/B of c5dbf7aedc4a0da1b71009ce70eea907 into 6d62711d02e749518a2b49c92a6bdd4b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:22,323 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:22,323 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/B, priority=13, startTime=1733496142262; duration=0sec 2024-12-06T14:42:22,331 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:22,331 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:B 2024-12-06T14:42:22,331 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:22,332 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:22,333 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/C is initiating minor compaction (all files) 2024-12-06T14:42:22,333 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/C in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,333 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c663f0f33d3b4d60aa5d96ba989ee69e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a06d44680cfd4b7b841b79fadf5064a8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/aa85d3b5f4b642f2a495134a802b4f1a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=35.7 K 2024-12-06T14:42:22,334 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c663f0f33d3b4d60aa5d96ba989ee69e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733496138372 2024-12-06T14:42:22,334 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a06d44680cfd4b7b841b79fadf5064a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733496138428 2024-12-06T14:42:22,335 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting aa85d3b5f4b642f2a495134a802b4f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733496140613 2024-12-06T14:42:22,351 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:22,351 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/d25e9f3cf6134c4e94c5e4f21f05e0c6 is 50, key is test_row_0/C:col10/1733496140613/Put/seqid=0 2024-12-06T14:42:22,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741973_1149 (size=12391) 2024-12-06T14:42:22,397 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/d25e9f3cf6134c4e94c5e4f21f05e0c6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d25e9f3cf6134c4e94c5e4f21f05e0c6 2024-12-06T14:42:22,408 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/C of c5dbf7aedc4a0da1b71009ce70eea907 into d25e9f3cf6134c4e94c5e4f21f05e0c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:22,408 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:22,408 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/C, priority=13, startTime=1733496142262; duration=0sec 2024-12-06T14:42:22,413 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:22,413 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:C 2024-12-06T14:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T14:42:22,569 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:22,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-06T14:42:22,572 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:22,573 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:22,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:22,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T14:42:22,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:22,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:22,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:22,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:22,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:22,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:22,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c3d7fdbc6a05459ca172ac491e5090f0_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496142638/Put/seqid=0 2024-12-06T14:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:22,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496202708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:22,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T14:42:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:22,731 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1 2024-12-06T14:42:22,738 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/A of c5dbf7aedc4a0da1b71009ce70eea907 into 30f4eb3ca26e4c8e9ef0ea6e9e337bf1(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:22,738 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:22,738 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/A, priority=13, startTime=1733496142262; duration=0sec 2024-12-06T14:42:22,738 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:22,738 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:A 2024-12-06T14:42:22,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741974_1150 (size=12304) 2024-12-06T14:42:22,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:22,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c3d7fdbc6a05459ca172ac491e5090f0_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c3d7fdbc6a05459ca172ac491e5090f0_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:22,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ccc6f58da4854d8c8cabd04b7c2afd48, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:22,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ccc6f58da4854d8c8cabd04b7c2afd48 is 175, key is test_row_0/A:col10/1733496142638/Put/seqid=0 2024-12-06T14:42:22,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741975_1151 (size=31105) 2024-12-06T14:42:22,778 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ccc6f58da4854d8c8cabd04b7c2afd48 2024-12-06T14:42:22,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/097aee10a2c74bf49532bc2149c5c093 is 50, key is test_row_0/B:col10/1733496142638/Put/seqid=0 2024-12-06T14:42:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496202818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:22,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741976_1152 (size=12151) 2024-12-06T14:42:22,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T14:42:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:22,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:23,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496203028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T14:42:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496203119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496203119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496203119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,122 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:23,122 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,124 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496203128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,130 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T14:42:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:23,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/097aee10a2c74bf49532bc2149c5c093 2024-12-06T14:42:23,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/0215c5d058bf4dd096da3047992cc6c2 is 50, key is test_row_0/C:col10/1733496142638/Put/seqid=0 2024-12-06T14:42:23,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741977_1153 (size=12151) 2024-12-06T14:42:23,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/0215c5d058bf4dd096da3047992cc6c2 2024-12-06T14:42:23,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ccc6f58da4854d8c8cabd04b7c2afd48 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48 2024-12-06T14:42:23,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48, entries=150, sequenceid=207, filesize=30.4 K 2024-12-06T14:42:23,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/097aee10a2c74bf49532bc2149c5c093 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/097aee10a2c74bf49532bc2149c5c093 2024-12-06T14:42:23,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/097aee10a2c74bf49532bc2149c5c093, entries=150, sequenceid=207, filesize=11.9 K 2024-12-06T14:42:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/0215c5d058bf4dd096da3047992cc6c2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0215c5d058bf4dd096da3047992cc6c2 2024-12-06T14:42:23,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496203340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T14:42:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:23,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0215c5d058bf4dd096da3047992cc6c2, entries=150, sequenceid=207, filesize=11.9 K 2024-12-06T14:42:23,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c5dbf7aedc4a0da1b71009ce70eea907 in 690ms, sequenceid=207, compaction requested=false 2024-12-06T14:42:23,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:23,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:23,500 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:23,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b18ab161ac794ac68668da38b08ce1ad_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496142688/Put/seqid=0 2024-12-06T14:42:23,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741978_1154 (size=12304) 2024-12-06T14:42:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496203926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:23,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:23,961 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b18ab161ac794ac68668da38b08ce1ad_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b18ab161ac794ac68668da38b08ce1ad_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fc787e535411499197a9fad83bd464c5, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:23,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fc787e535411499197a9fad83bd464c5 is 175, key is test_row_0/A:col10/1733496142688/Put/seqid=0 2024-12-06T14:42:23,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741979_1155 (size=31105) 2024-12-06T14:42:24,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496204038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:24,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496204248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:24,385 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fc787e535411499197a9fad83bd464c5 2024-12-06T14:42:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/8b98fa31b1e2466a9b9b9ff816e48976 is 50, key is test_row_0/B:col10/1733496142688/Put/seqid=0 2024-12-06T14:42:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741980_1156 (size=12151) 2024-12-06T14:42:24,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496204559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:24,810 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/8b98fa31b1e2466a9b9b9ff816e48976 2024-12-06T14:42:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/9d70543aada84f429e5c23eb44738cca is 50, key is test_row_0/C:col10/1733496142688/Put/seqid=0 2024-12-06T14:42:24,844 INFO [master/bb7ea3e46d9c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-06T14:42:24,844 INFO [master/bb7ea3e46d9c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-06T14:42:24,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741981_1157 (size=12151) 2024-12-06T14:42:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496205068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:25,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:25,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496205138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496205138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:25,142 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:25,142 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496205149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496205149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:25,152 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:25,152 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:25,263 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/9d70543aada84f429e5c23eb44738cca 2024-12-06T14:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/fc787e535411499197a9fad83bd464c5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5 2024-12-06T14:42:25,286 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5, entries=150, sequenceid=220, filesize=30.4 K 2024-12-06T14:42:25,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/8b98fa31b1e2466a9b9b9ff816e48976 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8b98fa31b1e2466a9b9b9ff816e48976 2024-12-06T14:42:25,294 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8b98fa31b1e2466a9b9b9ff816e48976, entries=150, sequenceid=220, filesize=11.9 K 2024-12-06T14:42:25,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/9d70543aada84f429e5c23eb44738cca as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/9d70543aada84f429e5c23eb44738cca 2024-12-06T14:42:25,306 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/9d70543aada84f429e5c23eb44738cca, entries=150, sequenceid=220, filesize=11.9 K 2024-12-06T14:42:25,308 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c5dbf7aedc4a0da1b71009ce70eea907 in 1808ms, sequenceid=220, compaction requested=true 2024-12-06T14:42:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-06T14:42:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-06T14:42:25,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-06T14:42:25,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7370 sec 2024-12-06T14:42:25,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.7460 sec 2024-12-06T14:42:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:26,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-06T14:42:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:26,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120641ac5a90319a43d2bb4b225a022deed7_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496206116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:26,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741982_1158 (size=14794) 2024-12-06T14:42:26,132 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:26,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120641ac5a90319a43d2bb4b225a022deed7_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120641ac5a90319a43d2bb4b225a022deed7_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:26,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0b3dadc62bd741ca8412a2cccf1cc069, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:26,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0b3dadc62bd741ca8412a2cccf1cc069 is 175, key is test_row_0/A:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:26,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741983_1159 (size=39749) 2024-12-06T14:42:26,148 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0b3dadc62bd741ca8412a2cccf1cc069 2024-12-06T14:42:26,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/554c8df071a442cb8d54e13aeff3106c is 50, key is test_row_0/B:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:26,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741984_1160 (size=12151) 2024-12-06T14:42:26,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496206238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:26,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496206449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:26,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/554c8df071a442cb8d54e13aeff3106c 2024-12-06T14:42:26,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/3a130215ac8849cd907fd68930b721b9 is 50, key is test_row_0/C:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:26,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741985_1161 (size=12151) 2024-12-06T14:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T14:42:26,728 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:26,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-06T14:42:26,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:26,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T14:42:26,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:26,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496206759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T14:42:26,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T14:42:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:26,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:26,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/3a130215ac8849cd907fd68930b721b9 2024-12-06T14:42:26,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/0b3dadc62bd741ca8412a2cccf1cc069 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069 2024-12-06T14:42:26,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069, entries=200, sequenceid=241, filesize=38.8 K 2024-12-06T14:42:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/554c8df071a442cb8d54e13aeff3106c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/554c8df071a442cb8d54e13aeff3106c 2024-12-06T14:42:26,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/554c8df071a442cb8d54e13aeff3106c, entries=150, sequenceid=241, filesize=11.9 K 2024-12-06T14:42:27,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/3a130215ac8849cd907fd68930b721b9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3a130215ac8849cd907fd68930b721b9 2024-12-06T14:42:27,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3a130215ac8849cd907fd68930b721b9, entries=150, sequenceid=241, filesize=11.9 K 2024-12-06T14:42:27,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c5dbf7aedc4a0da1b71009ce70eea907 in 942ms, sequenceid=241, compaction requested=true 2024-12-06T14:42:27,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:27,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:27,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:27,023 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:27,023 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:27,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:27,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:27,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:27,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:27,024 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:27,025 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/B is initiating minor compaction (all files) 2024-12-06T14:42:27,025 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/B in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:27,025 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6d62711d02e749518a2b49c92a6bdd4b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/097aee10a2c74bf49532bc2149c5c093, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8b98fa31b1e2466a9b9b9ff816e48976, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/554c8df071a442cb8d54e13aeff3106c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=47.7 K 2024-12-06T14:42:27,025 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133304 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:27,025 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d62711d02e749518a2b49c92a6bdd4b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733496140613 2024-12-06T14:42:27,025 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/A is initiating minor compaction (all files) 2024-12-06T14:42:27,025 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/A in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:27,025 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=130.2 K 2024-12-06T14:42:27,025 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:27,026 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069] 2024-12-06T14:42:27,026 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 097aee10a2c74bf49532bc2149c5c093, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733496140958 2024-12-06T14:42:27,026 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f4eb3ca26e4c8e9ef0ea6e9e337bf1, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733496140613 2024-12-06T14:42:27,027 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc6f58da4854d8c8cabd04b7c2afd48, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733496140958 2024-12-06T14:42:27,027 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b98fa31b1e2466a9b9b9ff816e48976, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733496142688 2024-12-06T14:42:27,027 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 554c8df071a442cb8d54e13aeff3106c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733496143909 2024-12-06T14:42:27,027 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc787e535411499197a9fad83bd464c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733496142688 2024-12-06T14:42:27,028 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3dadc62bd741ca8412a2cccf1cc069, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733496143909 2024-12-06T14:42:27,038 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:27,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T14:42:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:27,040 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:42:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:27,046 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:27,047 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/31581c6dba364deea5491425cb137888 is 50, key is test_row_0/B:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T14:42:27,048 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120698b82161110742588bdaf78768d2ab33_c5dbf7aedc4a0da1b71009ce70eea907 store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:27,050 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120698b82161110742588bdaf78768d2ab33_c5dbf7aedc4a0da1b71009ce70eea907, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:27,050 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120698b82161110742588bdaf78768d2ab33_c5dbf7aedc4a0da1b71009ce70eea907 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206213ffe63d44a4a96b0eeb95e4e48ed50_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496146115/Put/seqid=0 2024-12-06T14:42:27,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741986_1162 (size=12527) 2024-12-06T14:42:27,064 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/31581c6dba364deea5491425cb137888 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/31581c6dba364deea5491425cb137888 2024-12-06T14:42:27,069 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/B of c5dbf7aedc4a0da1b71009ce70eea907 into 31581c6dba364deea5491425cb137888(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:27,069 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:27,069 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/B, priority=12, startTime=1733496147023; duration=0sec 2024-12-06T14:42:27,069 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:27,069 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:B 2024-12-06T14:42:27,069 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:27,072 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:27,073 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/C is initiating minor compaction (all files) 2024-12-06T14:42:27,073 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/C in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:27,073 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d25e9f3cf6134c4e94c5e4f21f05e0c6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0215c5d058bf4dd096da3047992cc6c2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/9d70543aada84f429e5c23eb44738cca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3a130215ac8849cd907fd68930b721b9] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=47.7 K 2024-12-06T14:42:27,073 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d25e9f3cf6134c4e94c5e4f21f05e0c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733496140613 2024-12-06T14:42:27,073 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0215c5d058bf4dd096da3047992cc6c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733496140958 2024-12-06T14:42:27,074 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d70543aada84f429e5c23eb44738cca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733496142688 2024-12-06T14:42:27,074 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a130215ac8849cd907fd68930b721b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733496143909 2024-12-06T14:42:27,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741987_1163 (size=4469) 2024-12-06T14:42:27,079 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#A#compaction#135 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:27,080 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/7545daed59b84a3f9e0d7ac7103f9e5d is 175, key is test_row_0/A:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:27,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741988_1164 (size=12304) 2024-12-06T14:42:27,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741989_1165 (size=31481) 2024-12-06T14:42:27,103 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:27,103 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/d9ecdaaf7eaa459693a2665d5929a36e is 50, key is test_row_0/C:col10/1733496143909/Put/seqid=0 2024-12-06T14:42:27,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741990_1166 (size=12527) 2024-12-06T14:42:27,130 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/d9ecdaaf7eaa459693a2665d5929a36e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d9ecdaaf7eaa459693a2665d5929a36e 2024-12-06T14:42:27,138 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/C of c5dbf7aedc4a0da1b71009ce70eea907 into d9ecdaaf7eaa459693a2665d5929a36e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:27,138 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:27,139 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/C, priority=12, startTime=1733496147023; duration=0sec 2024-12-06T14:42:27,139 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:27,139 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:C 2024-12-06T14:42:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:27,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:27,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496147322, max timestamp=1733496147343Num uniques 859; 2024-12-06T14:42:27,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496147322, max timestamp=1733496147343Num uniques 859; 2024-12-06T14:42:27,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496147322, max timestamp=1733496147343Num uniques 859; 2024-12-06T14:42:27,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:27,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:27,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T14:42:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496207437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:27,498 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206213ffe63d44a4a96b0eeb95e4e48ed50_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206213ffe63d44a4a96b0eeb95e4e48ed50_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:27,499 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/7545daed59b84a3f9e0d7ac7103f9e5d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/7545daed59b84a3f9e0d7ac7103f9e5d 2024-12-06T14:42:27,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ad004317975c40378047750eba7806f4, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ad004317975c40378047750eba7806f4 is 175, key is test_row_0/A:col10/1733496146115/Put/seqid=0 2024-12-06T14:42:27,510 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/A of c5dbf7aedc4a0da1b71009ce70eea907 into 7545daed59b84a3f9e0d7ac7103f9e5d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:27,510 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:27,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741991_1167 (size=31105) 2024-12-06T14:42:27,510 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/A, priority=12, startTime=1733496147023; duration=0sec 2024-12-06T14:42:27,510 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:27,510 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:A 2024-12-06T14:42:27,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496207548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:27,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496207758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T14:42:27,913 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ad004317975c40378047750eba7806f4 2024-12-06T14:42:27,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f0a6d898fbb64aac824c359eff496860 is 50, key is test_row_0/B:col10/1733496146115/Put/seqid=0 2024-12-06T14:42:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741992_1168 (size=12151) 2024-12-06T14:42:28,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496208078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:28,335 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f0a6d898fbb64aac824c359eff496860 2024-12-06T14:42:28,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/11eb3ca59abe47a9927c426677cfa957 is 50, key is test_row_0/C:col10/1733496146115/Put/seqid=0 2024-12-06T14:42:28,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741993_1169 (size=12151) 2024-12-06T14:42:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496208589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:28,752 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/11eb3ca59abe47a9927c426677cfa957 2024-12-06T14:42:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/ad004317975c40378047750eba7806f4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4 2024-12-06T14:42:28,766 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4, entries=150, sequenceid=252, filesize=30.4 K 2024-12-06T14:42:28,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f0a6d898fbb64aac824c359eff496860 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f0a6d898fbb64aac824c359eff496860 2024-12-06T14:42:28,776 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f0a6d898fbb64aac824c359eff496860, entries=150, sequenceid=252, filesize=11.9 K 2024-12-06T14:42:28,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/11eb3ca59abe47a9927c426677cfa957 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/11eb3ca59abe47a9927c426677cfa957 2024-12-06T14:42:28,801 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/11eb3ca59abe47a9927c426677cfa957, entries=150, sequenceid=252, filesize=11.9 K 2024-12-06T14:42:28,803 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c5dbf7aedc4a0da1b71009ce70eea907 in 1762ms, sequenceid=252, compaction requested=false 2024-12-06T14:42:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-06T14:42:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-06T14:42:28,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-06T14:42:28,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0780 sec 2024-12-06T14:42:28,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0830 sec 2024-12-06T14:42:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T14:42:28,879 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:28,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-06T14:42:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T14:42:28,882 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:28,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:28,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T14:42:29,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:29,038 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:29,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120650e395829e90416cb941d38300de40d6_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496147434/Put/seqid=0 2024-12-06T14:42:29,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741994_1170 (size=12454) 2024-12-06T14:42:29,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:29,074 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120650e395829e90416cb941d38300de40d6_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120650e395829e90416cb941d38300de40d6_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/f2fd1fe0b36b4bc49a5cc7c7059607ca, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/f2fd1fe0b36b4bc49a5cc7c7059607ca is 175, key is test_row_0/A:col10/1733496147434/Put/seqid=0 2024-12-06T14:42:29,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741995_1171 (size=31255) 2024-12-06T14:42:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T14:42:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496209195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496209197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496209197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496209198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496209308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496209308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496209308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496209309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,490 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/f2fd1fe0b36b4bc49a5cc7c7059607ca 2024-12-06T14:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T14:42:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496209519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496209519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496209519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6e0d02eeea004db8bafa15b427966de0 is 50, key is test_row_0/B:col10/1733496147434/Put/seqid=0 2024-12-06T14:42:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496209519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741996_1172 (size=12301) 2024-12-06T14:42:29,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496209598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,601 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496209828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496209828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496209828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496209828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:29,955 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6e0d02eeea004db8bafa15b427966de0 2024-12-06T14:42:29,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/f9252a0dfbcc483199b6b1589361547e is 50, key is test_row_0/C:col10/1733496147434/Put/seqid=0 2024-12-06T14:42:29,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741997_1173 (size=12301) 2024-12-06T14:42:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T14:42:30,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496210338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496210338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496210338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496210338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:30,368 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/f9252a0dfbcc483199b6b1589361547e 2024-12-06T14:42:30,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/f2fd1fe0b36b4bc49a5cc7c7059607ca as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca 2024-12-06T14:42:30,377 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca, entries=150, sequenceid=290, filesize=30.5 K 2024-12-06T14:42:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6e0d02eeea004db8bafa15b427966de0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6e0d02eeea004db8bafa15b427966de0 2024-12-06T14:42:30,383 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6e0d02eeea004db8bafa15b427966de0, entries=150, sequenceid=290, filesize=12.0 K 2024-12-06T14:42:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/f9252a0dfbcc483199b6b1589361547e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/f9252a0dfbcc483199b6b1589361547e 2024-12-06T14:42:30,388 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/f9252a0dfbcc483199b6b1589361547e, entries=150, sequenceid=290, filesize=12.0 K 2024-12-06T14:42:30,389 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c5dbf7aedc4a0da1b71009ce70eea907 in 1351ms, sequenceid=290, compaction requested=true 2024-12-06T14:42:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-06T14:42:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-06T14:42:30,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-06T14:42:30,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-12-06T14:42:30,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5180 sec 2024-12-06T14:42:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T14:42:31,029 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:31,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-06T14:42:31,031 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T14:42:31,032 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:31,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T14:42:31,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T14:42:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:31,185 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b806cb3dee614045936f1d3d6aa474de_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496149195/Put/seqid=0 2024-12-06T14:42:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741998_1174 (size=12454) 2024-12-06T14:42:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T14:42:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:31,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496211371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496211374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496151366, max timestamp=1733496151374Num uniques 859; 2024-12-06T14:42:31,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496151366, max timestamp=1733496151374Num uniques 859; 2024-12-06T14:42:31,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:31,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:31,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496151366, max timestamp=1733496151374Num uniques 859; 2024-12-06T14:42:31,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:31,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496211408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496211413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496211479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496211479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496211518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496211528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496211619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,622 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b806cb3dee614045936f1d3d6aa474de_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b806cb3dee614045936f1d3d6aa474de_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:31,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/d06294f675d84d6e931015d948e59186, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:31,623 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/d06294f675d84d6e931015d948e59186 is 175, key is test_row_0/A:col10/1733496149195/Put/seqid=0 2024-12-06T14:42:31,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741999_1175 (size=31255) 2024-12-06T14:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T14:42:31,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496211688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496211688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496211730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496211738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:31,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496211998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496211998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,030 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/d06294f675d84d6e931015d948e59186 2024-12-06T14:42:32,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6f39ce34f94b4d63990c28bd3a526d70 is 50, key is test_row_0/B:col10/1733496149195/Put/seqid=0 2024-12-06T14:42:32,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496212038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742000_1176 (size=12301) 2024-12-06T14:42:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496212048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T14:42:32,451 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6f39ce34f94b4d63990c28bd3a526d70 2024-12-06T14:42:32,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b2503d9ad28e47fdb43ea1d889243437 is 50, key is test_row_0/C:col10/1733496149195/Put/seqid=0 2024-12-06T14:42:32,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742001_1177 (size=12301) 2024-12-06T14:42:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496212509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496212509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496212548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496212558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:32,871 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b2503d9ad28e47fdb43ea1d889243437 2024-12-06T14:42:32,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/d06294f675d84d6e931015d948e59186 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186 2024-12-06T14:42:32,888 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186, entries=150, sequenceid=301, filesize=30.5 K 2024-12-06T14:42:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/6f39ce34f94b4d63990c28bd3a526d70 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6f39ce34f94b4d63990c28bd3a526d70 2024-12-06T14:42:32,896 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6f39ce34f94b4d63990c28bd3a526d70, entries=150, sequenceid=301, filesize=12.0 K 2024-12-06T14:42:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/b2503d9ad28e47fdb43ea1d889243437 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b2503d9ad28e47fdb43ea1d889243437 2024-12-06T14:42:32,909 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b2503d9ad28e47fdb43ea1d889243437, entries=150, sequenceid=301, filesize=12.0 K 2024-12-06T14:42:32,912 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c5dbf7aedc4a0da1b71009ce70eea907 in 1726ms, sequenceid=301, compaction requested=true 2024-12-06T14:42:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-06T14:42:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-06T14:42:32,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-06T14:42:32,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-12-06T14:42:32,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8880 sec 2024-12-06T14:42:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T14:42:33,179 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:33,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-06T14:42:33,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T14:42:33,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:33,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T14:42:33,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T14:42:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:33,336 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-06T14:42:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:33,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:33,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:33,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067b9a8cc1509f4e30ba8284f860620f08_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496151411/Put/seqid=0 2024-12-06T14:42:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742002_1178 (size=12454) 2024-12-06T14:42:33,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:33,377 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067b9a8cc1509f4e30ba8284f860620f08_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067b9a8cc1509f4e30ba8284f860620f08_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:33,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/c35b411f86e74b1a909073570bd045bf, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:33,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/c35b411f86e74b1a909073570bd045bf is 175, key is test_row_0/A:col10/1733496151411/Put/seqid=0 2024-12-06T14:42:33,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742003_1179 (size=31255) 2024-12-06T14:42:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T14:42:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:33,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496213537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496213547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496213558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,560 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496213568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,572 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496213648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496213658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,793 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/c35b411f86e74b1a909073570bd045bf 2024-12-06T14:42:33,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/eaef0f7d1af0484ea443d27326864dc5 is 50, key is test_row_0/B:col10/1733496151411/Put/seqid=0 2024-12-06T14:42:33,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742004_1180 (size=12301) 2024-12-06T14:42:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T14:42:33,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496213858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496213882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:34,204 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/eaef0f7d1af0484ea443d27326864dc5 2024-12-06T14:42:34,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496214200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:34,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496214200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:34,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/2e2365c1957746f7ac99872d4b73a1da is 50, key is test_row_0/C:col10/1733496151411/Put/seqid=0 2024-12-06T14:42:34,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742005_1181 (size=12301) 2024-12-06T14:42:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T14:42:34,637 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/2e2365c1957746f7ac99872d4b73a1da 2024-12-06T14:42:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/c35b411f86e74b1a909073570bd045bf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf 2024-12-06T14:42:34,646 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf, entries=150, sequenceid=336, filesize=30.5 K 2024-12-06T14:42:34,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/eaef0f7d1af0484ea443d27326864dc5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/eaef0f7d1af0484ea443d27326864dc5 2024-12-06T14:42:34,651 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/eaef0f7d1af0484ea443d27326864dc5, entries=150, sequenceid=336, filesize=12.0 K 2024-12-06T14:42:34,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/2e2365c1957746f7ac99872d4b73a1da as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/2e2365c1957746f7ac99872d4b73a1da 2024-12-06T14:42:34,658 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/2e2365c1957746f7ac99872d4b73a1da, entries=150, sequenceid=336, filesize=12.0 K 2024-12-06T14:42:34,659 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c5dbf7aedc4a0da1b71009ce70eea907 in 1323ms, sequenceid=336, compaction requested=true 2024-12-06T14:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-06T14:42:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-06T14:42:34,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-06T14:42:34,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4770 sec 2024-12-06T14:42:34,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4830 sec 2024-12-06T14:42:34,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:42:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:34,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ee4d0e05d53d437d847a6ecfb8e3ef01_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496214780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:34,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496214782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:34,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742006_1182 (size=14994) 2024-12-06T14:42:34,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:34,795 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ee4d0e05d53d437d847a6ecfb8e3ef01_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ee4d0e05d53d437d847a6ecfb8e3ef01_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:34,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/8400d1d7d8d845a9972836c17ce4e282, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:34,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/8400d1d7d8d845a9972836c17ce4e282 is 175, key is test_row_0/A:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:34,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742007_1183 (size=39949) 2024-12-06T14:42:34,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/8400d1d7d8d845a9972836c17ce4e282 2024-12-06T14:42:34,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb is 50, key is test_row_0/B:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:34,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742008_1184 (size=12301) 2024-12-06T14:42:34,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496214888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:34,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496214888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:34,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496215099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496215099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb 2024-12-06T14:42:35,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/c38c8628cf16437da136ebee2939ebf3 is 50, key is test_row_0/C:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:35,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742009_1185 (size=12301) 2024-12-06T14:42:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T14:42:35,329 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:35,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-06T14:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:35,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:35,333 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:35,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496215408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496215408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:35,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T14:42:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:35,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36738 deadline: 1733496215578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1733496215578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,585 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,585 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36694 deadline: 1733496215638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T14:42:35,640 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in c5dbf7aedc4a0da1b71009ce70eea907 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:35,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:35,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/c38c8628cf16437da136ebee2939ebf3 2024-12-06T14:42:35,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/8400d1d7d8d845a9972836c17ce4e282 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282 2024-12-06T14:42:35,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282, entries=200, sequenceid=349, filesize=39.0 K 2024-12-06T14:42:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb 2024-12-06T14:42:35,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb, entries=150, sequenceid=349, filesize=12.0 K 2024-12-06T14:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/c38c8628cf16437da136ebee2939ebf3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c38c8628cf16437da136ebee2939ebf3 2024-12-06T14:42:35,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c38c8628cf16437da136ebee2939ebf3, entries=150, sequenceid=349, filesize=12.0 K 2024-12-06T14:42:35,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c5dbf7aedc4a0da1b71009ce70eea907 in 979ms, sequenceid=349, compaction requested=true 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:35,703 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-06T14:42:35,703 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:35,705 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 196300 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-06T14:42:35,705 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/A is initiating minor compaction (all files) 2024-12-06T14:42:35,705 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/A in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,706 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/7545daed59b84a3f9e0d7ac7103f9e5d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=191.7 K 2024-12-06T14:42:35,706 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,706 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/7545daed59b84a3f9e0d7ac7103f9e5d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282] 2024-12-06T14:42:35,706 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7545daed59b84a3f9e0d7ac7103f9e5d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733496143909 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad004317975c40378047750eba7806f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733496146095 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73882 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/B is initiating minor compaction (all files) 2024-12-06T14:42:35,707 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/B in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,707 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/31581c6dba364deea5491425cb137888, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f0a6d898fbb64aac824c359eff496860, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6e0d02eeea004db8bafa15b427966de0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6f39ce34f94b4d63990c28bd3a526d70, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/eaef0f7d1af0484ea443d27326864dc5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=72.2 K 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2fd1fe0b36b4bc49a5cc7c7059607ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733496147405 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 31581c6dba364deea5491425cb137888, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733496143909 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d06294f675d84d6e931015d948e59186, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733496149193 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a6d898fbb64aac824c359eff496860, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733496146095 2024-12-06T14:42:35,707 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35b411f86e74b1a909073570bd045bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733496151404 2024-12-06T14:42:35,708 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0d02eeea004db8bafa15b427966de0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733496147405 2024-12-06T14:42:35,708 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8400d1d7d8d845a9972836c17ce4e282, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733496153529 2024-12-06T14:42:35,708 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f39ce34f94b4d63990c28bd3a526d70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733496149193 2024-12-06T14:42:35,708 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting eaef0f7d1af0484ea443d27326864dc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733496151404 2024-12-06T14:42:35,709 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f4bdb2c3aad74e93a4cf6bfdb677d8fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733496153535 2024-12-06T14:42:35,731 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#B#compaction#153 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:35,731 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:35,731 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/616a68ad882c442aa9dfe131b41cfad6 is 50, key is test_row_0/B:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:35,737 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412061100872bdfc748f7beac514b6d0869d7_c5dbf7aedc4a0da1b71009ce70eea907 store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:35,740 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412061100872bdfc748f7beac514b6d0869d7_c5dbf7aedc4a0da1b71009ce70eea907, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:35,740 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061100872bdfc748f7beac514b6d0869d7_c5dbf7aedc4a0da1b71009ce70eea907 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:35,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742011_1187 (size=4469) 2024-12-06T14:42:35,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742010_1186 (size=12881) 2024-12-06T14:42:35,757 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#A#compaction#154 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:35,758 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/183f812dda4e41d2bd0ac4cbe252a297 is 175, key is test_row_0/A:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:35,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742012_1188 (size=31835) 2024-12-06T14:42:35,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T14:42:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,793 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:42:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:35,798 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/183f812dda4e41d2bd0ac4cbe252a297 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/183f812dda4e41d2bd0ac4cbe252a297 2024-12-06T14:42:35,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120661dd091dd8c2462b86af1817b5308169_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496154777/Put/seqid=0 2024-12-06T14:42:35,813 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/A of c5dbf7aedc4a0da1b71009ce70eea907 into 183f812dda4e41d2bd0ac4cbe252a297(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:35,813 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:35,813 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/A, priority=10, startTime=1733496155703; duration=0sec 2024-12-06T14:42:35,814 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:35,814 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:A 2024-12-06T14:42:35,814 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-06T14:42:35,816 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73882 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-06T14:42:35,816 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/C is initiating minor compaction (all files) 2024-12-06T14:42:35,816 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/C in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:35,816 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d9ecdaaf7eaa459693a2665d5929a36e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/11eb3ca59abe47a9927c426677cfa957, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/f9252a0dfbcc483199b6b1589361547e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b2503d9ad28e47fdb43ea1d889243437, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/2e2365c1957746f7ac99872d4b73a1da, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c38c8628cf16437da136ebee2939ebf3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=72.2 K 2024-12-06T14:42:35,817 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ecdaaf7eaa459693a2665d5929a36e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733496143909 2024-12-06T14:42:35,817 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11eb3ca59abe47a9927c426677cfa957, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733496146095 2024-12-06T14:42:35,817 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9252a0dfbcc483199b6b1589361547e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733496147405 2024-12-06T14:42:35,818 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2503d9ad28e47fdb43ea1d889243437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733496149193 2024-12-06T14:42:35,818 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e2365c1957746f7ac99872d4b73a1da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733496151404 2024-12-06T14:42:35,818 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38c8628cf16437da136ebee2939ebf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733496153535 2024-12-06T14:42:35,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742013_1189 (size=12454) 2024-12-06T14:42:35,839 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:35,839 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/6b19edf661634b72a377d09e70ad21a5 is 50, key is test_row_0/C:col10/1733496153535/Put/seqid=0 2024-12-06T14:42:35,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742014_1190 (size=12881) 2024-12-06T14:42:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:35,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. as already flushing 2024-12-06T14:42:35,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496215947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496215948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:36,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496216058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:36,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496216058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:36,164 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/616a68ad882c442aa9dfe131b41cfad6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/616a68ad882c442aa9dfe131b41cfad6 2024-12-06T14:42:36,176 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/B of c5dbf7aedc4a0da1b71009ce70eea907 into 616a68ad882c442aa9dfe131b41cfad6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:36,176 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:36,176 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/B, priority=10, startTime=1733496155703; duration=0sec 2024-12-06T14:42:36,177 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:36,177 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:B 2024-12-06T14:42:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:36,235 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120661dd091dd8c2462b86af1817b5308169_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120661dd091dd8c2462b86af1817b5308169_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/bbe4d747b4b94d18b4a9a7dd6eee9689, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/bbe4d747b4b94d18b4a9a7dd6eee9689 is 175, key is test_row_0/A:col10/1733496154777/Put/seqid=0 2024-12-06T14:42:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742015_1191 (size=31255) 2024-12-06T14:42:36,252 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/6b19edf661634b72a377d09e70ad21a5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/6b19edf661634b72a377d09e70ad21a5 2024-12-06T14:42:36,257 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/C of c5dbf7aedc4a0da1b71009ce70eea907 into 6b19edf661634b72a377d09e70ad21a5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:36,257 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:36,257 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/C, priority=10, startTime=1733496155703; duration=0sec 2024-12-06T14:42:36,257 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:36,257 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:C 2024-12-06T14:42:36,258 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-12-06T14:42:36,258 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:36,258 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,258 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:36,263 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-12-06T14:42:36,263 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:36,263 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,263 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:36,264 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-12-06T14:42:36,264 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:36,264 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,264 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:36,265 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-12-06T14:42:36,265 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:36,265 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,265 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:36,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496216268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496216269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:36,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496216578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:36,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496216578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:36,646 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/bbe4d747b4b94d18b4a9a7dd6eee9689 2024-12-06T14:42:36,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/617c62727d7b4c68a222c62a36722cbe is 50, key is test_row_0/B:col10/1733496154777/Put/seqid=0 2024-12-06T14:42:36,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742016_1192 (size=12301) 2024-12-06T14:42:37,064 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/617c62727d7b4c68a222c62a36722cbe 2024-12-06T14:42:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/33b6e2111c1a4996be2b055b9281b6eb is 50, key is test_row_0/C:col10/1733496154777/Put/seqid=0 2024-12-06T14:42:37,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:37,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36708 deadline: 1733496217088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36716 deadline: 1733496217088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:42:37,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742017_1193 (size=12301) 2024-12-06T14:42:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:37,491 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/33b6e2111c1a4996be2b055b9281b6eb 2024-12-06T14:42:37,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/bbe4d747b4b94d18b4a9a7dd6eee9689 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689 2024-12-06T14:42:37,502 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689, entries=150, sequenceid=368, filesize=30.5 K 2024-12-06T14:42:37,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/617c62727d7b4c68a222c62a36722cbe as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/617c62727d7b4c68a222c62a36722cbe 2024-12-06T14:42:37,508 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/617c62727d7b4c68a222c62a36722cbe, entries=150, sequenceid=368, filesize=12.0 K 2024-12-06T14:42:37,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/33b6e2111c1a4996be2b055b9281b6eb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/33b6e2111c1a4996be2b055b9281b6eb 2024-12-06T14:42:37,512 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/33b6e2111c1a4996be2b055b9281b6eb, entries=150, sequenceid=368, filesize=12.0 K 2024-12-06T14:42:37,513 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c5dbf7aedc4a0da1b71009ce70eea907 in 1720ms, sequenceid=368, compaction requested=false 2024-12-06T14:42:37,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:37,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:37,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-06T14:42:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-06T14:42:37,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-06T14:42:37,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-12-06T14:42:37,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1860 sec 2024-12-06T14:42:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:38,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:42:38,159 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-12-06T14:42:38,159 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-12-06T14:42:38,159 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:38,159 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:38,159 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:38,159 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:38,159 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:38,159 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:38,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:38,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:38,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206133dfc9e7a5242e4aac97b312b827935_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_0/A:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:38,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742018_1194 (size=12454) 2024-12-06T14:42:38,546 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T14:42:38,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:38,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206133dfc9e7a5242e4aac97b312b827935_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206133dfc9e7a5242e4aac97b312b827935_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:38,591 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/9ba0019041874cbeb0fa862b738025c5, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:38,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/9ba0019041874cbeb0fa862b738025c5 is 175, key is test_row_0/A:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:38,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742019_1195 (size=31255) 2024-12-06T14:42:38,600 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/9ba0019041874cbeb0fa862b738025c5 2024-12-06T14:42:38,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/7468a5ba8d524b498eb4d6ab4be2c564 is 50, key is test_row_0/B:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:38,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742020_1196 (size=12301) 2024-12-06T14:42:39,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/7468a5ba8d524b498eb4d6ab4be2c564 2024-12-06T14:42:39,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/3188688be8db44f5b51e3f20fc2857c5 is 50, key is test_row_0/C:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:39,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742021_1197 (size=12301) 2024-12-06T14:42:39,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/3188688be8db44f5b51e3f20fc2857c5 2024-12-06T14:42:39,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/9ba0019041874cbeb0fa862b738025c5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5 2024-12-06T14:42:39,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5, entries=150, sequenceid=386, filesize=30.5 K 2024-12-06T14:42:39,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/7468a5ba8d524b498eb4d6ab4be2c564 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/7468a5ba8d524b498eb4d6ab4be2c564 2024-12-06T14:42:39,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/7468a5ba8d524b498eb4d6ab4be2c564, entries=150, sequenceid=386, filesize=12.0 K 2024-12-06T14:42:39,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/3188688be8db44f5b51e3f20fc2857c5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3188688be8db44f5b51e3f20fc2857c5 2024-12-06T14:42:39,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3188688be8db44f5b51e3f20fc2857c5, entries=150, sequenceid=386, filesize=12.0 K 2024-12-06T14:42:39,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for c5dbf7aedc4a0da1b71009ce70eea907 in 1309ms, sequenceid=386, compaction requested=true 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:39,469 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:39,469 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5dbf7aedc4a0da1b71009ce70eea907:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:39,470 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:39,470 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/A is initiating minor compaction (all files) 2024-12-06T14:42:39,470 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/A in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:39,470 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/183f812dda4e41d2bd0ac4cbe252a297, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=92.1 K 2024-12-06T14:42:39,470 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:39,470 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/183f812dda4e41d2bd0ac4cbe252a297, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5] 2024-12-06T14:42:39,471 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:39,471 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/B is initiating minor compaction (all files) 2024-12-06T14:42:39,471 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/B in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:39,471 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/616a68ad882c442aa9dfe131b41cfad6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/617c62727d7b4c68a222c62a36722cbe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/7468a5ba8d524b498eb4d6ab4be2c564] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=36.6 K 2024-12-06T14:42:39,471 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 183f812dda4e41d2bd0ac4cbe252a297, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733496153535 2024-12-06T14:42:39,472 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 616a68ad882c442aa9dfe131b41cfad6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733496153535 2024-12-06T14:42:39,472 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe4d747b4b94d18b4a9a7dd6eee9689, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733496154767 2024-12-06T14:42:39,472 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 617c62727d7b4c68a222c62a36722cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733496154767 2024-12-06T14:42:39,472 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ba0019041874cbeb0fa862b738025c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733496155947 2024-12-06T14:42:39,472 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 7468a5ba8d524b498eb4d6ab4be2c564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733496155947 2024-12-06T14:42:39,497 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#B#compaction#162 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:39,497 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/2789fabe676c444e82238d304597bf75 is 50, key is test_row_0/B:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:39,503 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:39,509 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412066d90efef13af49dda5fc2f0779e15e42_c5dbf7aedc4a0da1b71009ce70eea907 store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T14:42:39,525 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:39,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742022_1198 (size=12983) 2024-12-06T14:42:39,575 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412066d90efef13af49dda5fc2f0779e15e42_c5dbf7aedc4a0da1b71009ce70eea907, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:39,575 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066d90efef13af49dda5fc2f0779e15e42_c5dbf7aedc4a0da1b71009ce70eea907 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:39,601 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-12-06T14:42:39,601 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:39,601 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:39,601 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:39,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:39,604 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-12-06T14:42:39,604 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:39,604 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:39,604 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:39,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:39,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742023_1199 (size=4469) 2024-12-06T14:42:39,946 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/2789fabe676c444e82238d304597bf75 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/2789fabe676c444e82238d304597bf75 2024-12-06T14:42:39,957 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/B of c5dbf7aedc4a0da1b71009ce70eea907 into 2789fabe676c444e82238d304597bf75(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:39,957 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:39,957 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/B, priority=13, startTime=1733496159469; duration=0sec 2024-12-06T14:42:39,957 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:39,958 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:B 2024-12-06T14:42:39,958 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:39,960 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:39,960 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): c5dbf7aedc4a0da1b71009ce70eea907/C is initiating minor compaction (all files) 2024-12-06T14:42:39,960 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5dbf7aedc4a0da1b71009ce70eea907/C in TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:39,960 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/6b19edf661634b72a377d09e70ad21a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/33b6e2111c1a4996be2b055b9281b6eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3188688be8db44f5b51e3f20fc2857c5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp, totalSize=36.6 K 2024-12-06T14:42:39,961 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b19edf661634b72a377d09e70ad21a5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733496153535 2024-12-06T14:42:39,961 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b6e2111c1a4996be2b055b9281b6eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733496154767 2024-12-06T14:42:39,962 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3188688be8db44f5b51e3f20fc2857c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733496155947 2024-12-06T14:42:39,982 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#C#compaction#164 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:39,983 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/0ee5077064b347be915ed2e6c7092e0d is 50, key is test_row_0/C:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:40,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742024_1200 (size=12983) 2024-12-06T14:42:40,009 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5dbf7aedc4a0da1b71009ce70eea907#A#compaction#163 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:40,010 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/f436cdd5b44541f4a48982f6c2172cbc is 175, key is test_row_0/A:col10/1733496158099/Put/seqid=0 2024-12-06T14:42:40,014 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/0ee5077064b347be915ed2e6c7092e0d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0ee5077064b347be915ed2e6c7092e0d 2024-12-06T14:42:40,027 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/C of c5dbf7aedc4a0da1b71009ce70eea907 into 0ee5077064b347be915ed2e6c7092e0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:40,027 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:40,027 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/C, priority=13, startTime=1733496159469; duration=0sec 2024-12-06T14:42:40,028 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:40,028 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:C 2024-12-06T14:42:40,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742025_1201 (size=31937) 2024-12-06T14:42:40,454 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/f436cdd5b44541f4a48982f6c2172cbc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f436cdd5b44541f4a48982f6c2172cbc 2024-12-06T14:42:40,464 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5dbf7aedc4a0da1b71009ce70eea907/A of c5dbf7aedc4a0da1b71009ce70eea907 into f436cdd5b44541f4a48982f6c2172cbc(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:40,464 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5dbf7aedc4a0da1b71009ce70eea907: 2024-12-06T14:42:40,464 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., storeName=c5dbf7aedc4a0da1b71009ce70eea907/A, priority=13, startTime=1733496159469; duration=0sec 2024-12-06T14:42:40,464 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:40,464 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5dbf7aedc4a0da1b71009ce70eea907:A 2024-12-06T14:42:45,679 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-12-06T14:42:45,680 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:42:45,680 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:45,680 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:45,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3553 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3566 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1569 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4706 rows 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1566 2024-12-06T14:42:45,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4697 rows 2024-12-06T14:42:45,680 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:42:45,680 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:42:45,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:45,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:45,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:42:45,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T14:42:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T14:42:45,686 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496165685"}]},"ts":"1733496165685"} 2024-12-06T14:42:45,688 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T14:42:45,688 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T14:42:45,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:42:45,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, UNASSIGN}] 2024-12-06T14:42:45,697 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, UNASSIGN 2024-12-06T14:42:45,698 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:45,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, UNASSIGN because future has completed 2024-12-06T14:42:45,699 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:42:45,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:42:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T14:42:45,851 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing c5dbf7aedc4a0da1b71009ce70eea907, disabling compactions & flushes 2024-12-06T14:42:45,851 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. after waiting 0 ms 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:45,851 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing c5dbf7aedc4a0da1b71009ce70eea907 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=A 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=B 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c5dbf7aedc4a0da1b71009ce70eea907, store=C 2024-12-06T14:42:45,851 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:45,857 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f033249a8f6f410498b58e93db41cd71_c5dbf7aedc4a0da1b71009ce70eea907 is 50, key is test_row_2/A:col10/1733496165678/Put/seqid=0 2024-12-06T14:42:45,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742026_1202 (size=7374) 2024-12-06T14:42:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T14:42:46,262 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:46,266 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f033249a8f6f410498b58e93db41cd71_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f033249a8f6f410498b58e93db41cd71_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:46,267 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/b994c25c2d17497988178884fe1eb538, store: [table=TestAcidGuarantees family=A region=c5dbf7aedc4a0da1b71009ce70eea907] 2024-12-06T14:42:46,268 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/b994c25c2d17497988178884fe1eb538 is 175, key is test_row_2/A:col10/1733496165678/Put/seqid=0 2024-12-06T14:42:46,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742027_1203 (size=13865) 2024-12-06T14:42:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T14:42:46,673 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/b994c25c2d17497988178884fe1eb538 2024-12-06T14:42:46,684 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/c15d14270a6d45998acf6a38b4701aa9 is 50, key is test_row_2/B:col10/1733496165678/Put/seqid=0 2024-12-06T14:42:46,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742028_1204 (size=7415) 2024-12-06T14:42:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T14:42:47,089 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/c15d14270a6d45998acf6a38b4701aa9 2024-12-06T14:42:47,104 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a479efe837a04a659dcc14417a2dd30d is 50, key is test_row_2/C:col10/1733496165678/Put/seqid=0 2024-12-06T14:42:47,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742029_1205 (size=7415) 2024-12-06T14:42:47,508 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a479efe837a04a659dcc14417a2dd30d 2024-12-06T14:42:47,514 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/A/b994c25c2d17497988178884fe1eb538 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/b994c25c2d17497988178884fe1eb538 2024-12-06T14:42:47,520 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/b994c25c2d17497988178884fe1eb538, entries=50, sequenceid=395, filesize=13.5 K 2024-12-06T14:42:47,521 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/B/c15d14270a6d45998acf6a38b4701aa9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c15d14270a6d45998acf6a38b4701aa9 2024-12-06T14:42:47,527 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c15d14270a6d45998acf6a38b4701aa9, entries=50, sequenceid=395, filesize=7.2 K 2024-12-06T14:42:47,528 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/.tmp/C/a479efe837a04a659dcc14417a2dd30d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a479efe837a04a659dcc14417a2dd30d 2024-12-06T14:42:47,535 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a479efe837a04a659dcc14417a2dd30d, entries=50, sequenceid=395, filesize=7.2 K 2024-12-06T14:42:47,536 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c5dbf7aedc4a0da1b71009ce70eea907 in 1685ms, sequenceid=395, compaction requested=false 2024-12-06T14:42:47,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/708e557cba22477db9a180685c948038, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/7545daed59b84a3f9e0d7ac7103f9e5d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/183f812dda4e41d2bd0ac4cbe252a297, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5] to archive 2024-12-06T14:42:47,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:42:47,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fee18ded821e4780bc69ee336b087a77 2024-12-06T14:42:47,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/cf118bd5f78c49778a6f0bed23c46ea9 2024-12-06T14:42:47,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/940f561fa4384c8993397f5637a680dd 2024-12-06T14:42:47,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0866e13f759b46229e1099501de4f82c 2024-12-06T14:42:47,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/708e557cba22477db9a180685c948038 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/708e557cba22477db9a180685c948038 2024-12-06T14:42:47,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0378669dba084bf687f03805ecd0ca42 2024-12-06T14:42:47,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/30f4eb3ca26e4c8e9ef0ea6e9e337bf1 2024-12-06T14:42:47,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/09e5557af648407c9a030fbb6446878b 2024-12-06T14:42:47,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ccc6f58da4854d8c8cabd04b7c2afd48 2024-12-06T14:42:47,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/fc787e535411499197a9fad83bd464c5 2024-12-06T14:42:47,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/0b3dadc62bd741ca8412a2cccf1cc069 2024-12-06T14:42:47,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/7545daed59b84a3f9e0d7ac7103f9e5d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/7545daed59b84a3f9e0d7ac7103f9e5d 2024-12-06T14:42:47,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/ad004317975c40378047750eba7806f4 2024-12-06T14:42:47,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f2fd1fe0b36b4bc49a5cc7c7059607ca 2024-12-06T14:42:47,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/d06294f675d84d6e931015d948e59186 2024-12-06T14:42:47,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/c35b411f86e74b1a909073570bd045bf 2024-12-06T14:42:47,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/8400d1d7d8d845a9972836c17ce4e282 2024-12-06T14:42:47,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/183f812dda4e41d2bd0ac4cbe252a297 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/183f812dda4e41d2bd0ac4cbe252a297 2024-12-06T14:42:47,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/bbe4d747b4b94d18b4a9a7dd6eee9689 2024-12-06T14:42:47,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/9ba0019041874cbeb0fa862b738025c5 2024-12-06T14:42:47,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/e0aef5e7c048434eb78024aa7b24a948, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/59f2859c6a5d43f9b676d5b22583efa9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c6864abaf7d04da09ea9948f3854d50a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f3b8a5a9f3c346e38c3bbb19e8e54c64, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/91a1a76ae91a4deb8c485abf12229705, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8220bd2ca0f14890b282e694e2af49a7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6d62711d02e749518a2b49c92a6bdd4b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/82f2eed446f2498aadea580effd99632, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/097aee10a2c74bf49532bc2149c5c093, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8b98fa31b1e2466a9b9b9ff816e48976, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/31581c6dba364deea5491425cb137888, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/554c8df071a442cb8d54e13aeff3106c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f0a6d898fbb64aac824c359eff496860, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6e0d02eeea004db8bafa15b427966de0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6f39ce34f94b4d63990c28bd3a526d70, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/eaef0f7d1af0484ea443d27326864dc5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/616a68ad882c442aa9dfe131b41cfad6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/617c62727d7b4c68a222c62a36722cbe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/7468a5ba8d524b498eb4d6ab4be2c564] to archive 2024-12-06T14:42:47,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:42:47,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/e0aef5e7c048434eb78024aa7b24a948 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/e0aef5e7c048434eb78024aa7b24a948 2024-12-06T14:42:47,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/59f2859c6a5d43f9b676d5b22583efa9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/59f2859c6a5d43f9b676d5b22583efa9 2024-12-06T14:42:47,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c6864abaf7d04da09ea9948f3854d50a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c6864abaf7d04da09ea9948f3854d50a 2024-12-06T14:42:47,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f3b8a5a9f3c346e38c3bbb19e8e54c64 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f3b8a5a9f3c346e38c3bbb19e8e54c64 2024-12-06T14:42:47,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/91a1a76ae91a4deb8c485abf12229705 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/91a1a76ae91a4deb8c485abf12229705 2024-12-06T14:42:47,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8220bd2ca0f14890b282e694e2af49a7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8220bd2ca0f14890b282e694e2af49a7 2024-12-06T14:42:47,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6d62711d02e749518a2b49c92a6bdd4b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6d62711d02e749518a2b49c92a6bdd4b 2024-12-06T14:42:47,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/82f2eed446f2498aadea580effd99632 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/82f2eed446f2498aadea580effd99632 2024-12-06T14:42:47,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/097aee10a2c74bf49532bc2149c5c093 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/097aee10a2c74bf49532bc2149c5c093 2024-12-06T14:42:47,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8b98fa31b1e2466a9b9b9ff816e48976 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/8b98fa31b1e2466a9b9b9ff816e48976 2024-12-06T14:42:47,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/31581c6dba364deea5491425cb137888 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/31581c6dba364deea5491425cb137888 2024-12-06T14:42:47,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/554c8df071a442cb8d54e13aeff3106c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/554c8df071a442cb8d54e13aeff3106c 2024-12-06T14:42:47,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f0a6d898fbb64aac824c359eff496860 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f0a6d898fbb64aac824c359eff496860 2024-12-06T14:42:47,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6e0d02eeea004db8bafa15b427966de0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6e0d02eeea004db8bafa15b427966de0 2024-12-06T14:42:47,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6f39ce34f94b4d63990c28bd3a526d70 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/6f39ce34f94b4d63990c28bd3a526d70 2024-12-06T14:42:47,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/eaef0f7d1af0484ea443d27326864dc5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/eaef0f7d1af0484ea443d27326864dc5 2024-12-06T14:42:47,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/616a68ad882c442aa9dfe131b41cfad6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/616a68ad882c442aa9dfe131b41cfad6 2024-12-06T14:42:47,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/f4bdb2c3aad74e93a4cf6bfdb677d8fb 2024-12-06T14:42:47,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/617c62727d7b4c68a222c62a36722cbe to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/617c62727d7b4c68a222c62a36722cbe 2024-12-06T14:42:47,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/7468a5ba8d524b498eb4d6ab4be2c564 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/7468a5ba8d524b498eb4d6ab4be2c564 2024-12-06T14:42:47,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/5b448b3a66b34db69ce05ebf53e5c59b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b670de1a25604e1ba11db2d287ad46b8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b743583cc3e1495182d058a185b0bde1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c663f0f33d3b4d60aa5d96ba989ee69e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a3b9dbcd423447a2bd50c1b10c45cdc6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a06d44680cfd4b7b841b79fadf5064a8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d25e9f3cf6134c4e94c5e4f21f05e0c6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/aa85d3b5f4b642f2a495134a802b4f1a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0215c5d058bf4dd096da3047992cc6c2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/9d70543aada84f429e5c23eb44738cca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d9ecdaaf7eaa459693a2665d5929a36e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3a130215ac8849cd907fd68930b721b9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/11eb3ca59abe47a9927c426677cfa957, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/f9252a0dfbcc483199b6b1589361547e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b2503d9ad28e47fdb43ea1d889243437, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/2e2365c1957746f7ac99872d4b73a1da, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/6b19edf661634b72a377d09e70ad21a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c38c8628cf16437da136ebee2939ebf3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/33b6e2111c1a4996be2b055b9281b6eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3188688be8db44f5b51e3f20fc2857c5] to archive 2024-12-06T14:42:47,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:42:47,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/5b448b3a66b34db69ce05ebf53e5c59b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/5b448b3a66b34db69ce05ebf53e5c59b 2024-12-06T14:42:47,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b670de1a25604e1ba11db2d287ad46b8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b670de1a25604e1ba11db2d287ad46b8 2024-12-06T14:42:47,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b743583cc3e1495182d058a185b0bde1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b743583cc3e1495182d058a185b0bde1 2024-12-06T14:42:47,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c663f0f33d3b4d60aa5d96ba989ee69e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c663f0f33d3b4d60aa5d96ba989ee69e 2024-12-06T14:42:47,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a3b9dbcd423447a2bd50c1b10c45cdc6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a3b9dbcd423447a2bd50c1b10c45cdc6 2024-12-06T14:42:47,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a06d44680cfd4b7b841b79fadf5064a8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a06d44680cfd4b7b841b79fadf5064a8 2024-12-06T14:42:47,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d25e9f3cf6134c4e94c5e4f21f05e0c6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d25e9f3cf6134c4e94c5e4f21f05e0c6 2024-12-06T14:42:47,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/aa85d3b5f4b642f2a495134a802b4f1a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/aa85d3b5f4b642f2a495134a802b4f1a 2024-12-06T14:42:47,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0215c5d058bf4dd096da3047992cc6c2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0215c5d058bf4dd096da3047992cc6c2 2024-12-06T14:42:47,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/9d70543aada84f429e5c23eb44738cca to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/9d70543aada84f429e5c23eb44738cca 2024-12-06T14:42:47,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d9ecdaaf7eaa459693a2665d5929a36e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/d9ecdaaf7eaa459693a2665d5929a36e 2024-12-06T14:42:47,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3a130215ac8849cd907fd68930b721b9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3a130215ac8849cd907fd68930b721b9 2024-12-06T14:42:47,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/11eb3ca59abe47a9927c426677cfa957 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/11eb3ca59abe47a9927c426677cfa957 2024-12-06T14:42:47,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/f9252a0dfbcc483199b6b1589361547e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/f9252a0dfbcc483199b6b1589361547e 2024-12-06T14:42:47,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b2503d9ad28e47fdb43ea1d889243437 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/b2503d9ad28e47fdb43ea1d889243437 2024-12-06T14:42:47,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/2e2365c1957746f7ac99872d4b73a1da to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/2e2365c1957746f7ac99872d4b73a1da 2024-12-06T14:42:47,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/6b19edf661634b72a377d09e70ad21a5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/6b19edf661634b72a377d09e70ad21a5 2024-12-06T14:42:47,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c38c8628cf16437da136ebee2939ebf3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/c38c8628cf16437da136ebee2939ebf3 2024-12-06T14:42:47,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/33b6e2111c1a4996be2b055b9281b6eb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/33b6e2111c1a4996be2b055b9281b6eb 2024-12-06T14:42:47,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3188688be8db44f5b51e3f20fc2857c5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/3188688be8db44f5b51e3f20fc2857c5 2024-12-06T14:42:47,685 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/recovered.edits/398.seqid, newMaxSeqId=398, maxSeqId=4 2024-12-06T14:42:47,686 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. 2024-12-06T14:42:47,686 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for c5dbf7aedc4a0da1b71009ce70eea907: Waiting for close lock at 1733496165851Running coprocessor pre-close hooks at 1733496165851Disabling compacts and flushes for region at 1733496165851Disabling writes for close at 1733496165851Obtaining lock to block concurrent updates at 1733496165851Preparing flush snapshotting stores in c5dbf7aedc4a0da1b71009ce70eea907 at 1733496165851Finished memstore snapshotting TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733496165851Flushing stores of TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907. at 1733496165852 (+1 ms)Flushing c5dbf7aedc4a0da1b71009ce70eea907/A: creating writer at 1733496165852Flushing mob file c5dbf7aedc4a0da1b71009ce70eea907/A: appending metadata at 1733496165857 (+5 ms)Flushing mob file c5dbf7aedc4a0da1b71009ce70eea907/A: closing flushed file at 1733496165857Flushing c5dbf7aedc4a0da1b71009ce70eea907/A: appending metadata at 1733496166267 (+410 ms)Flushing c5dbf7aedc4a0da1b71009ce70eea907/A: closing flushed file at 1733496166267Flushing c5dbf7aedc4a0da1b71009ce70eea907/B: creating writer at 1733496166679 (+412 ms)Flushing c5dbf7aedc4a0da1b71009ce70eea907/B: appending metadata at 1733496166683 (+4 ms)Flushing c5dbf7aedc4a0da1b71009ce70eea907/B: closing flushed file at 1733496166683Flushing c5dbf7aedc4a0da1b71009ce70eea907/C: creating writer at 1733496167098 (+415 ms)Flushing c5dbf7aedc4a0da1b71009ce70eea907/C: appending metadata at 1733496167103 (+5 ms)Flushing c5dbf7aedc4a0da1b71009ce70eea907/C: closing flushed file at 1733496167103Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@513e2b30: reopening flushed file at 1733496167513 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31645da2: reopening flushed file at 1733496167520 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b38ab57: reopening flushed file at 1733496167527 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c5dbf7aedc4a0da1b71009ce70eea907 in 1685ms, sequenceid=395, compaction requested=false at 1733496167536 (+9 ms)Writing region close event to WAL at 1733496167681 (+145 ms)Running coprocessor post-close hooks at 1733496167686 (+5 ms)Closed at 1733496167686 2024-12-06T14:42:47,690 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,690 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c5dbf7aedc4a0da1b71009ce70eea907, regionState=CLOSED 2024-12-06T14:42:47,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:42:47,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-06T14:42:47,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure c5dbf7aedc4a0da1b71009ce70eea907, server=bb7ea3e46d9c,35831,1733496100253 in 1.9930 sec 2024-12-06T14:42:47,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-06T14:42:47,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5dbf7aedc4a0da1b71009ce70eea907, UNASSIGN in 1.9990 sec 2024-12-06T14:42:47,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-06T14:42:47,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0100 sec 2024-12-06T14:42:47,703 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496167702"}]},"ts":"1733496167702"} 2024-12-06T14:42:47,704 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T14:42:47,704 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T14:42:47,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0240 sec 2024-12-06T14:42:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T14:42:47,829 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:47,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T14:42:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,832 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T14:42:47,833 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/recovered.edits] 2024-12-06T14:42:47,844 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/b994c25c2d17497988178884fe1eb538 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/b994c25c2d17497988178884fe1eb538 2024-12-06T14:42:47,845 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f436cdd5b44541f4a48982f6c2172cbc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/A/f436cdd5b44541f4a48982f6c2172cbc 2024-12-06T14:42:47,849 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/2789fabe676c444e82238d304597bf75 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/2789fabe676c444e82238d304597bf75 2024-12-06T14:42:47,851 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c15d14270a6d45998acf6a38b4701aa9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/B/c15d14270a6d45998acf6a38b4701aa9 2024-12-06T14:42:47,854 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0ee5077064b347be915ed2e6c7092e0d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/0ee5077064b347be915ed2e6c7092e0d 2024-12-06T14:42:47,857 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a479efe837a04a659dcc14417a2dd30d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/C/a479efe837a04a659dcc14417a2dd30d 2024-12-06T14:42:47,862 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/recovered.edits/398.seqid to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907/recovered.edits/398.seqid 2024-12-06T14:42:47,862 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,862 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T14:42:47,863 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T14:42:47,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T14:42:47,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206133dfc9e7a5242e4aac97b312b827935_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206133dfc9e7a5242e4aac97b312b827935_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,873 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120613cddeb265114ea689c556ed0d9dc762_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120613cddeb265114ea689c556ed0d9dc762_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,877 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206213ffe63d44a4a96b0eeb95e4e48ed50_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206213ffe63d44a4a96b0eeb95e4e48ed50_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206289b04db511b456ba6d74ecc354e6828_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206289b04db511b456ba6d74ecc354e6828_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,882 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120641ac5a90319a43d2bb4b225a022deed7_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120641ac5a90319a43d2bb4b225a022deed7_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,884 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064d9935a668b24600895bfed0062046ef_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064d9935a668b24600895bfed0062046ef_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,887 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120650e395829e90416cb941d38300de40d6_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120650e395829e90416cb941d38300de40d6_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,888 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120661dd091dd8c2462b86af1817b5308169_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120661dd091dd8c2462b86af1817b5308169_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,889 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067b9a8cc1509f4e30ba8284f860620f08_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067b9a8cc1509f4e30ba8284f860620f08_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,891 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b18ab161ac794ac68668da38b08ce1ad_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b18ab161ac794ac68668da38b08ce1ad_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,896 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b558c5c1b6f645ec8560d3d9dfe6f80c_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b558c5c1b6f645ec8560d3d9dfe6f80c_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,898 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b806cb3dee614045936f1d3d6aa474de_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b806cb3dee614045936f1d3d6aa474de_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,900 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ba8891f1525446ce8579164d84e439a4_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ba8891f1525446ce8579164d84e439a4_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,902 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c3d7fdbc6a05459ca172ac491e5090f0_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c3d7fdbc6a05459ca172ac491e5090f0_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,904 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e5ad0504a0424403afe10e8d641854e1_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e5ad0504a0424403afe10e8d641854e1_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,906 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ee4d0e05d53d437d847a6ecfb8e3ef01_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ee4d0e05d53d437d847a6ecfb8e3ef01_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,908 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f033249a8f6f410498b58e93db41cd71_c5dbf7aedc4a0da1b71009ce70eea907 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f033249a8f6f410498b58e93db41cd71_c5dbf7aedc4a0da1b71009ce70eea907 2024-12-06T14:42:47,909 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T14:42:47,914 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,917 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T14:42:47,921 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T14:42:47,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T14:42:47,923 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733496167923"}]},"ts":"9223372036854775807"} 2024-12-06T14:42:47,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T14:42:47,926 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c5dbf7aedc4a0da1b71009ce70eea907, NAME => 'TestAcidGuarantees,,1733496132999.c5dbf7aedc4a0da1b71009ce70eea907.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T14:42:47,926 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T14:42:47,927 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733496167926"}]},"ts":"9223372036854775807"} 2024-12-06T14:42:47,929 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T14:42:47,930 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-12-06T14:42:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T14:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T14:42:47,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:47,955 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 297) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5777 (was 5718) - AvailableMemoryMB LEAK? - 2024-12-06T14:42:47,968 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5776 2024-12-06T14:42:47,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:42:47,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:42:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T14:42:47,973 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:42:47,973 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:47,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-06T14:42:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-06T14:42:47,976 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:42:47,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742030_1206 (size=1061) 2024-12-06T14:42:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-06T14:42:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-06T14:42:48,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:42:48,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742031_1207 (size=53) 2024-12-06T14:42:48,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:42:48,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4828895d58f7749c326fda40500dce6a, disabling compactions & flushes 2024-12-06T14:42:48,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. after waiting 0 ms 2024-12-06T14:42:48,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4828895d58f7749c326fda40500dce6a: Waiting for close lock at 1733496168415Disabling compacts and flushes for region at 1733496168415Disabling writes for close at 1733496168415Writing region close event to WAL at 1733496168415Closed at 1733496168415 2024-12-06T14:42:48,417 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:42:48,417 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733496168417"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496168417"}]},"ts":"1733496168417"} 2024-12-06T14:42:48,419 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:42:48,420 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:42:48,421 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496168421"}]},"ts":"1733496168421"} 2024-12-06T14:42:48,423 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T14:42:48,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, ASSIGN}] 2024-12-06T14:42:48,426 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, ASSIGN 2024-12-06T14:42:48,427 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=false 2024-12-06T14:42:48,577 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=4828895d58f7749c326fda40500dce6a, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:48,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, ASSIGN because future has completed 2024-12-06T14:42:48,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:42:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-06T14:42:48,735 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,735 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:42:48,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:42:48,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,738 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,740 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:48,740 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4828895d58f7749c326fda40500dce6a columnFamilyName A 2024-12-06T14:42:48,740 DEBUG [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:48,741 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.HStore(327): Store=4828895d58f7749c326fda40500dce6a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:48,741 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,743 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:48,743 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4828895d58f7749c326fda40500dce6a columnFamilyName B 2024-12-06T14:42:48,743 DEBUG [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:48,744 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.HStore(327): Store=4828895d58f7749c326fda40500dce6a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:48,744 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,745 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:42:48,746 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4828895d58f7749c326fda40500dce6a columnFamilyName C 2024-12-06T14:42:48,746 DEBUG [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:42:48,747 INFO [StoreOpener-4828895d58f7749c326fda40500dce6a-1 {}] regionserver.HStore(327): Store=4828895d58f7749c326fda40500dce6a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:42:48,747 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,747 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,748 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,748 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,750 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,750 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,750 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:42:48,752 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,755 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:42:48,756 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 4828895d58f7749c326fda40500dce6a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70029989, jitterRate=0.043528154492378235}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:42:48,756 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:48,757 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 4828895d58f7749c326fda40500dce6a: Running coprocessor pre-open hook at 1733496168736Writing region info on filesystem at 1733496168736Initializing all the Stores at 1733496168737 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496168738 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496168738Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496168738Cleaning up temporary data from old regions at 1733496168750 (+12 ms)Running coprocessor post-open hooks at 1733496168756 (+6 ms)Region opened successfully at 1733496168757 (+1 ms) 2024-12-06T14:42:48,758 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., pid=61, masterSystemTime=1733496168731 2024-12-06T14:42:48,760 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,761 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:48,761 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=4828895d58f7749c326fda40500dce6a, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:48,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:42:48,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-06T14:42:48,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 in 185 msec 2024-12-06T14:42:48,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-06T14:42:48,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, ASSIGN in 342 msec 2024-12-06T14:42:48,770 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:42:48,770 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496168770"}]},"ts":"1733496168770"} 2024-12-06T14:42:48,772 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T14:42:48,774 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:42:48,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 804 msec 2024-12-06T14:42:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-06T14:42:49,109 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:49,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba7ffab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,111 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c626002, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34937ab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,115 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:42:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:42:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d72d05c 2024-12-06T14:42:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:42:49,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:42:49,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11961dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,125 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779aea0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,127 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f8e779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48345386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e386816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,132 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513782ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ddee8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,135 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbeb097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,137 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403e7385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5138abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258c8742, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8220aac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3128e873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,155 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761f8ff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,163 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41169d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,164 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af52d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289dbcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,167 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22491eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,169 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b7d0f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d509c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,173 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344ac1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478bceb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,175 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f4436fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7933620e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63eadf7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:42:49,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:42:49,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:42:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:42:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:42:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e00fad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:42:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:42:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:42:49,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:42:49,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e7a0f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:42:49,182 DEBUG [Thread-990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,182 DEBUG [Thread-992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-06T14:42:49,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T14:42:49,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:49,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:49,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,188 DEBUG [Thread-1001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,189 DEBUG [Thread-986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,189 DEBUG [Thread-988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,189 DEBUG [Thread-994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,189 DEBUG [Thread-1003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,189 DEBUG [Thread-986 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,190 DEBUG [Thread-986 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,191 DEBUG [Thread-1005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,191 DEBUG [Thread-997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,194 DEBUG [Thread-999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:42:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:42:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:42:49,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:49,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:42:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:42:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:42:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:49,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496229230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496229231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496229231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496229233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496229234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/db59a63c08be4a139ec5190b6aa68a57 is 50, key is test_row_0/A:col10/1733496169197/Put/seqid=0 2024-12-06T14:42:49,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742032_1208 (size=12001) 2024-12-06T14:42:49,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/db59a63c08be4a139ec5190b6aa68a57 2024-12-06T14:42:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T14:42:49,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/d30052fd3dab46b79bd96a2b84e7e383 is 50, key is test_row_0/B:col10/1733496169197/Put/seqid=0 2024-12-06T14:42:49,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496229339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496229339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496229339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496229348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496229348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742033_1209 (size=12001) 2024-12-06T14:42:49,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/d30052fd3dab46b79bd96a2b84e7e383 2024-12-06T14:42:49,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e0a7731da1e8423e87f0212a5e0e159d is 50, key is test_row_0/C:col10/1733496169197/Put/seqid=0 2024-12-06T14:42:49,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742034_1210 (size=12001) 2024-12-06T14:42:49,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T14:42:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496229558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496229558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496229559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496229559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496229559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T14:42:49,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e0a7731da1e8423e87f0212a5e0e159d 2024-12-06T14:42:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/db59a63c08be4a139ec5190b6aa68a57 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/db59a63c08be4a139ec5190b6aa68a57 2024-12-06T14:42:49,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/db59a63c08be4a139ec5190b6aa68a57, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:42:49,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/d30052fd3dab46b79bd96a2b84e7e383 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/d30052fd3dab46b79bd96a2b84e7e383 2024-12-06T14:42:49,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/d30052fd3dab46b79bd96a2b84e7e383, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:42:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e0a7731da1e8423e87f0212a5e0e159d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e0a7731da1e8423e87f0212a5e0e159d 2024-12-06T14:42:49,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e0a7731da1e8423e87f0212a5e0e159d, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:42:49,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4828895d58f7749c326fda40500dce6a in 631ms, sequenceid=14, compaction requested=false 2024-12-06T14:42:49,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:49,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T14:42:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:49,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/df263c7f66154d28bb53e22c61095f1e is 50, key is test_row_0/A:col10/1733496169871/Put/seqid=0 2024-12-06T14:42:49,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496229879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496229882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496229883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496229884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496229884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:49,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742035_1211 (size=12001) 2024-12-06T14:42:49,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/df263c7f66154d28bb53e22c61095f1e 2024-12-06T14:42:49,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/14faec9c51ca4d868f38228eefd19e0b is 50, key is test_row_0/B:col10/1733496169871/Put/seqid=0 2024-12-06T14:42:49,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742036_1212 (size=12001) 2024-12-06T14:42:49,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/14faec9c51ca4d868f38228eefd19e0b 2024-12-06T14:42:49,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:49,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:49,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:49,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:49,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a1efafc8cd194af1bfae87b76ec045d8 is 50, key is test_row_0/C:col10/1733496169871/Put/seqid=0 2024-12-06T14:42:49,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742037_1213 (size=12001) 2024-12-06T14:42:49,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496229988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496229999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496229999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496229999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496229999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:50,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496230198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496230209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496230209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496230209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496230209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:50,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:50,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:50,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T14:42:50,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a1efafc8cd194af1bfae87b76ec045d8 2024-12-06T14:42:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/df263c7f66154d28bb53e22c61095f1e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/df263c7f66154d28bb53e22c61095f1e 2024-12-06T14:42:50,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/df263c7f66154d28bb53e22c61095f1e, entries=150, sequenceid=35, filesize=11.7 K 2024-12-06T14:42:50,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/14faec9c51ca4d868f38228eefd19e0b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14faec9c51ca4d868f38228eefd19e0b 2024-12-06T14:42:50,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14faec9c51ca4d868f38228eefd19e0b, entries=150, sequenceid=35, filesize=11.7 K 2024-12-06T14:42:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a1efafc8cd194af1bfae87b76ec045d8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a1efafc8cd194af1bfae87b76ec045d8 2024-12-06T14:42:50,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a1efafc8cd194af1bfae87b76ec045d8, entries=150, sequenceid=35, filesize=11.7 K 2024-12-06T14:42:50,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 4828895d58f7749c326fda40500dce6a in 537ms, sequenceid=35, compaction requested=false 2024-12-06T14:42:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:50,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T14:42:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:50,411 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:42:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:50,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/fcbc8ddd283b4b4e8241e1e233aadcc2 is 50, key is test_row_0/A:col10/1733496169880/Put/seqid=0 2024-12-06T14:42:50,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742038_1214 (size=12001) 2024-12-06T14:42:50,424 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/fcbc8ddd283b4b4e8241e1e233aadcc2 2024-12-06T14:42:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/84dbda18f81b415385a1e412c99f6657 is 50, key is test_row_0/B:col10/1733496169880/Put/seqid=0 2024-12-06T14:42:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:50,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:50,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742039_1215 (size=12001) 2024-12-06T14:42:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496230544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496170541, max timestamp=1733496170549Num uniques 859; 2024-12-06T14:42:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496170541, max timestamp=1733496170549Num uniques 859; 2024-12-06T14:42:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:50,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496170541, max timestamp=1733496170549Num uniques 859; 2024-12-06T14:42:50,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496230579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496230579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496230581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496230583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496230659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496230688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496230688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496230698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496230698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496230868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496230898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496230898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496230909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496230909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:50,930 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/84dbda18f81b415385a1e412c99f6657 2024-12-06T14:42:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/555f98f6ffbd4c37a30413fff46d2a1a is 50, key is test_row_0/C:col10/1733496169880/Put/seqid=0 2024-12-06T14:42:50,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742040_1216 (size=12001) 2024-12-06T14:42:50,962 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/555f98f6ffbd4c37a30413fff46d2a1a 2024-12-06T14:42:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/fcbc8ddd283b4b4e8241e1e233aadcc2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/fcbc8ddd283b4b4e8241e1e233aadcc2 2024-12-06T14:42:50,973 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/fcbc8ddd283b4b4e8241e1e233aadcc2, entries=150, sequenceid=46, filesize=11.7 K 2024-12-06T14:42:50,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/84dbda18f81b415385a1e412c99f6657 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/84dbda18f81b415385a1e412c99f6657 2024-12-06T14:42:50,979 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/84dbda18f81b415385a1e412c99f6657, entries=150, sequenceid=46, filesize=11.7 K 2024-12-06T14:42:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/555f98f6ffbd4c37a30413fff46d2a1a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/555f98f6ffbd4c37a30413fff46d2a1a 2024-12-06T14:42:50,986 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/555f98f6ffbd4c37a30413fff46d2a1a, entries=150, sequenceid=46, filesize=11.7 K 2024-12-06T14:42:50,986 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4828895d58f7749c326fda40500dce6a in 575ms, sequenceid=46, compaction requested=true 2024-12-06T14:42:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:50,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-06T14:42:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-06T14:42:50,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-06T14:42:50,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-12-06T14:42:50,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8080 sec 2024-12-06T14:42:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:51,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-06T14:42:51,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:51,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:51,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/d4fc467b66974979a27fd8a563aa182c is 50, key is test_row_0/A:col10/1733496170580/Put/seqid=0 2024-12-06T14:42:51,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742041_1217 (size=12001) 2024-12-06T14:42:51,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/d4fc467b66974979a27fd8a563aa182c 2024-12-06T14:42:51,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496231216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496231219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496231219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496231222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496231224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/b639d9028f424fc584e770dd2ca198c6 is 50, key is test_row_0/B:col10/1733496170580/Put/seqid=0 2024-12-06T14:42:51,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742042_1218 (size=12001) 2024-12-06T14:42:51,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/b639d9028f424fc584e770dd2ca198c6 2024-12-06T14:42:51,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1fba0aa077f14169b178beb5cca8fa60 is 50, key is test_row_0/C:col10/1733496170580/Put/seqid=0 2024-12-06T14:42:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T14:42:51,328 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:51,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-06T14:42:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T14:42:51,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:51,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496231328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:51,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:51,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742043_1219 (size=12001) 2024-12-06T14:42:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496231338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496231338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T14:42:51,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:51,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:51,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496231539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496231548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496231549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:51,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:51,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T14:42:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496231728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496231728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1fba0aa077f14169b178beb5cca8fa60 2024-12-06T14:42:51,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/d4fc467b66974979a27fd8a563aa182c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/d4fc467b66974979a27fd8a563aa182c 2024-12-06T14:42:51,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/d4fc467b66974979a27fd8a563aa182c, entries=150, sequenceid=83, filesize=11.7 K 2024-12-06T14:42:51,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/b639d9028f424fc584e770dd2ca198c6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b639d9028f424fc584e770dd2ca198c6 2024-12-06T14:42:51,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b639d9028f424fc584e770dd2ca198c6, entries=150, sequenceid=83, filesize=11.7 K 2024-12-06T14:42:51,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1fba0aa077f14169b178beb5cca8fa60 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1fba0aa077f14169b178beb5cca8fa60 2024-12-06T14:42:51,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:51,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:51,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1fba0aa077f14169b178beb5cca8fa60, entries=150, sequenceid=83, filesize=11.7 K 2024-12-06T14:42:51,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4828895d58f7749c326fda40500dce6a in 613ms, sequenceid=83, compaction requested=true 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:51,795 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:51,795 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:51,797 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:51,797 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:42:51,797 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,797 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/db59a63c08be4a139ec5190b6aa68a57, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/df263c7f66154d28bb53e22c61095f1e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/fcbc8ddd283b4b4e8241e1e233aadcc2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/d4fc467b66974979a27fd8a563aa182c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=46.9 K 2024-12-06T14:42:51,797 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:51,797 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:42:51,797 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,798 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/d30052fd3dab46b79bd96a2b84e7e383, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14faec9c51ca4d868f38228eefd19e0b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/84dbda18f81b415385a1e412c99f6657, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b639d9028f424fc584e770dd2ca198c6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=46.9 K 2024-12-06T14:42:51,798 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d30052fd3dab46b79bd96a2b84e7e383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496169197 2024-12-06T14:42:51,799 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 14faec9c51ca4d868f38228eefd19e0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733496169215 2024-12-06T14:42:51,799 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting db59a63c08be4a139ec5190b6aa68a57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496169197 2024-12-06T14:42:51,799 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 84dbda18f81b415385a1e412c99f6657, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733496169880 2024-12-06T14:42:51,799 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting df263c7f66154d28bb53e22c61095f1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733496169215 2024-12-06T14:42:51,800 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcbc8ddd283b4b4e8241e1e233aadcc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733496169880 2024-12-06T14:42:51,800 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b639d9028f424fc584e770dd2ca198c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496170578 2024-12-06T14:42:51,800 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4fc467b66974979a27fd8a563aa182c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496170578 2024-12-06T14:42:51,835 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:51,836 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:51,836 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/98678d6622f04550a56b4643867539f1 is 50, key is test_row_0/A:col10/1733496170580/Put/seqid=0 2024-12-06T14:42:51,836 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/36b54a855404493abc81945aa922027a is 50, key is test_row_0/B:col10/1733496170580/Put/seqid=0 2024-12-06T14:42:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:51,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:51,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742044_1220 (size=12139) 2024-12-06T14:42:51,876 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/98678d6622f04550a56b4643867539f1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/98678d6622f04550a56b4643867539f1 2024-12-06T14:42:51,883 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 98678d6622f04550a56b4643867539f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:51,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:51,883 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=12, startTime=1733496171795; duration=0sec 2024-12-06T14:42:51,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:51,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:42:51,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:51,886 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:51,886 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:42:51,886 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,886 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e0a7731da1e8423e87f0212a5e0e159d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a1efafc8cd194af1bfae87b76ec045d8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/555f98f6ffbd4c37a30413fff46d2a1a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1fba0aa077f14169b178beb5cca8fa60] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=46.9 K 2024-12-06T14:42:51,886 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0a7731da1e8423e87f0212a5e0e159d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496169197 2024-12-06T14:42:51,887 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1efafc8cd194af1bfae87b76ec045d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733496169215 2024-12-06T14:42:51,887 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 555f98f6ffbd4c37a30413fff46d2a1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733496169880 2024-12-06T14:42:51,888 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fba0aa077f14169b178beb5cca8fa60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496170578 2024-12-06T14:42:51,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/53862087b7194a6280ab50f1eab197c8 is 50, key is test_row_0/A:col10/1733496171851/Put/seqid=0 2024-12-06T14:42:51,900 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:51,901 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/93665f6451c14c2a8eca599e71698f5e is 50, key is test_row_0/C:col10/1733496170580/Put/seqid=0 2024-12-06T14:42:51,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742045_1221 (size=12139) 2024-12-06T14:42:51,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742047_1223 (size=12139) 2024-12-06T14:42:51,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496171881, max timestamp=1733496171910Num uniques 859; 2024-12-06T14:42:51,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:51,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496171897, max timestamp=1733496171910Num uniques 859; 2024-12-06T14:42:51,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T14:42:51,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496171897, max timestamp=1733496171910Num uniques 859; 2024-12-06T14:42:51,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T14:42:51,922 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/93665f6451c14c2a8eca599e71698f5e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/93665f6451c14c2a8eca599e71698f5e 2024-12-06T14:42:51,930 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into 93665f6451c14c2a8eca599e71698f5e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:51,931 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:51,931 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=12, startTime=1733496171795; duration=0sec 2024-12-06T14:42:51,931 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:51,931 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:42:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742046_1222 (size=12001) 2024-12-06T14:42:51,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/53862087b7194a6280ab50f1eab197c8 2024-12-06T14:42:51,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:51,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T14:42:51,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/83714f8c28bf476ba8ed36c136923063 is 50, key is test_row_0/B:col10/1733496171851/Put/seqid=0 2024-12-06T14:42:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496231956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496231961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:51,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496231963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742048_1224 (size=12001) 2024-12-06T14:42:52,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496232068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496232078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496232078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:52,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496232278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496232288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496232288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,325 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/36b54a855404493abc81945aa922027a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/36b54a855404493abc81945aa922027a 2024-12-06T14:42:52,332 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into 36b54a855404493abc81945aa922027a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:52,332 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:52,332 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=12, startTime=1733496171795; duration=0sec 2024-12-06T14:42:52,333 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:52,333 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:42:52,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/83714f8c28bf476ba8ed36c136923063 2024-12-06T14:42:52,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:52,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1215914dcc6945ce9b12abda07220c23 is 50, key is test_row_0/C:col10/1733496171851/Put/seqid=0 2024-12-06T14:42:52,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742049_1225 (size=12001) 2024-12-06T14:42:52,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1215914dcc6945ce9b12abda07220c23 2024-12-06T14:42:52,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/53862087b7194a6280ab50f1eab197c8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/53862087b7194a6280ab50f1eab197c8 2024-12-06T14:42:52,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/53862087b7194a6280ab50f1eab197c8, entries=150, sequenceid=94, filesize=11.7 K 2024-12-06T14:42:52,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/83714f8c28bf476ba8ed36c136923063 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/83714f8c28bf476ba8ed36c136923063 2024-12-06T14:42:52,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/83714f8c28bf476ba8ed36c136923063, entries=150, sequenceid=94, filesize=11.7 K 2024-12-06T14:42:52,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1215914dcc6945ce9b12abda07220c23 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1215914dcc6945ce9b12abda07220c23 2024-12-06T14:42:52,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1215914dcc6945ce9b12abda07220c23, entries=150, sequenceid=94, filesize=11.7 K 2024-12-06T14:42:52,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=135.63 KB/138880 for 4828895d58f7749c326fda40500dce6a in 614ms, sequenceid=94, compaction requested=false 2024-12-06T14:42:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T14:42:52,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T14:42:52,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,566 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-12-06T14:42:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:52,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/a2f826e56944477c9d562497efaa7ae5 is 50, key is test_row_0/A:col10/1733496171950/Put/seqid=0 2024-12-06T14:42:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:52,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:52,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742050_1226 (size=12151) 2024-12-06T14:42:52,607 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/a2f826e56944477c9d562497efaa7ae5 2024-12-06T14:42:52,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496232607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496232612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e0e28816dbe24024af870217f9729042 is 50, key is test_row_0/B:col10/1733496171950/Put/seqid=0 2024-12-06T14:42:52,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496232613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742051_1227 (size=12151) 2024-12-06T14:42:52,653 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e0e28816dbe24024af870217f9729042 2024-12-06T14:42:52,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/34e67ae21a04407db33e3d628b4cfef2 is 50, key is test_row_0/C:col10/1733496171950/Put/seqid=0 2024-12-06T14:42:52,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742052_1228 (size=12151) 2024-12-06T14:42:52,710 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/34e67ae21a04407db33e3d628b4cfef2 2024-12-06T14:42:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/a2f826e56944477c9d562497efaa7ae5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/a2f826e56944477c9d562497efaa7ae5 2024-12-06T14:42:52,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496232718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,722 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/a2f826e56944477c9d562497efaa7ae5, entries=150, sequenceid=134, filesize=11.9 K 2024-12-06T14:42:52,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e0e28816dbe24024af870217f9729042 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e0e28816dbe24024af870217f9729042 2024-12-06T14:42:52,729 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e0e28816dbe24024af870217f9729042, entries=150, sequenceid=134, filesize=11.9 K 2024-12-06T14:42:52,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/34e67ae21a04407db33e3d628b4cfef2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/34e67ae21a04407db33e3d628b4cfef2 2024-12-06T14:42:52,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496232729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496232730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,738 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/34e67ae21a04407db33e3d628b4cfef2, entries=150, sequenceid=134, filesize=11.9 K 2024-12-06T14:42:52,739 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138880, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for 4828895d58f7749c326fda40500dce6a in 173ms, sequenceid=134, compaction requested=true 2024-12-06T14:42:52,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:52,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:52,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-06T14:42:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-06T14:42:52,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-06T14:42:52,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-12-06T14:42:52,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4150 sec 2024-12-06T14:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:52,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:42:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:52,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/bb2da9b6b48a48158ee7d4a6e7bd872e is 50, key is test_row_0/A:col10/1733496172590/Put/seqid=0 2024-12-06T14:42:52,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742053_1229 (size=12151) 2024-12-06T14:42:52,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/bb2da9b6b48a48158ee7d4a6e7bd872e 2024-12-06T14:42:52,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/a07c23f772ba4438bde93d702acc1f9a is 50, key is test_row_0/B:col10/1733496172590/Put/seqid=0 2024-12-06T14:42:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496232803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496232806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742054_1230 (size=12151) 2024-12-06T14:42:52,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496232919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496232919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496232928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496232939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496232939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496233129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496233129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,229 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:42:53,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/a07c23f772ba4438bde93d702acc1f9a 2024-12-06T14:42:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496233239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496233248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496233248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/3357006aa19847d2a44a068b17c74258 is 50, key is test_row_0/C:col10/1733496172590/Put/seqid=0 2024-12-06T14:42:53,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742055_1231 (size=12151) 2024-12-06T14:42:53,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496233439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496233439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T14:42:53,479 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:53,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-06T14:42:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:53,481 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:53,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:53,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:53,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/3357006aa19847d2a44a068b17c74258 2024-12-06T14:42:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/bb2da9b6b48a48158ee7d4a6e7bd872e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/bb2da9b6b48a48158ee7d4a6e7bd872e 2024-12-06T14:42:53,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/bb2da9b6b48a48158ee7d4a6e7bd872e, entries=150, sequenceid=146, filesize=11.9 K 2024-12-06T14:42:53,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/a07c23f772ba4438bde93d702acc1f9a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a07c23f772ba4438bde93d702acc1f9a 2024-12-06T14:42:53,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a07c23f772ba4438bde93d702acc1f9a, entries=150, sequenceid=146, filesize=11.9 K 2024-12-06T14:42:53,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/3357006aa19847d2a44a068b17c74258 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/3357006aa19847d2a44a068b17c74258 2024-12-06T14:42:53,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/3357006aa19847d2a44a068b17c74258, entries=150, sequenceid=146, filesize=11.9 K 2024-12-06T14:42:53,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4828895d58f7749c326fda40500dce6a in 964ms, sequenceid=146, compaction requested=true 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:53,716 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:53,716 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:53,721 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:53,721 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:42:53,721 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,721 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/98678d6622f04550a56b4643867539f1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/53862087b7194a6280ab50f1eab197c8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/a2f826e56944477c9d562497efaa7ae5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/bb2da9b6b48a48158ee7d4a6e7bd872e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=47.3 K 2024-12-06T14:42:53,722 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98678d6622f04550a56b4643867539f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496170578 2024-12-06T14:42:53,722 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:53,722 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:42:53,722 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53862087b7194a6280ab50f1eab197c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496171219 2024-12-06T14:42:53,722 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,722 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/36b54a855404493abc81945aa922027a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/83714f8c28bf476ba8ed36c136923063, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e0e28816dbe24024af870217f9729042, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a07c23f772ba4438bde93d702acc1f9a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=47.3 K 2024-12-06T14:42:53,723 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f826e56944477c9d562497efaa7ae5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733496171950 2024-12-06T14:42:53,723 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b54a855404493abc81945aa922027a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496170578 2024-12-06T14:42:53,724 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 83714f8c28bf476ba8ed36c136923063, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496171219 2024-12-06T14:42:53,724 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2da9b6b48a48158ee7d4a6e7bd872e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733496172590 2024-12-06T14:42:53,724 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e28816dbe24024af870217f9729042, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733496171950 2024-12-06T14:42:53,725 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a07c23f772ba4438bde93d702acc1f9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733496172590 2024-12-06T14:42:53,751 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:53,752 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/783e306ee19f4db7b2c48aa5362b77b8 is 50, key is test_row_0/A:col10/1733496172590/Put/seqid=0 2024-12-06T14:42:53,758 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:53,759 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/2fa0685c6b884efd8ca2b033f07160a5 is 50, key is test_row_0/B:col10/1733496172590/Put/seqid=0 2024-12-06T14:42:53,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496172788, max timestamp=1733496173756Num uniques 859; 2024-12-06T14:42:53,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496172788, max timestamp=1733496173756Num uniques 859; 2024-12-06T14:42:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:53,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-12-06T14:42:53,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496172788, max timestamp=1733496173763Num uniques 859; 2024-12-06T14:42:53,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:53,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742056_1232 (size=12425) 2024-12-06T14:42:53,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:53,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:53,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:53,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:53,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:53,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:53,783 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/783e306ee19f4db7b2c48aa5362b77b8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/783e306ee19f4db7b2c48aa5362b77b8 2024-12-06T14:42:53,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/6a0fd27d318a463fb23d24b982646645 is 50, key is test_row_0/A:col10/1733496173756/Put/seqid=0 2024-12-06T14:42:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742057_1233 (size=12425) 2024-12-06T14:42:53,833 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/2fa0685c6b884efd8ca2b033f07160a5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/2fa0685c6b884efd8ca2b033f07160a5 2024-12-06T14:42:53,837 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 783e306ee19f4db7b2c48aa5362b77b8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:53,837 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:53,837 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=12, startTime=1733496173716; duration=0sec 2024-12-06T14:42:53,837 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:53,837 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:42:53,837 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:42:53,839 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:42:53,840 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:42:53,840 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,840 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/93665f6451c14c2a8eca599e71698f5e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1215914dcc6945ce9b12abda07220c23, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/34e67ae21a04407db33e3d628b4cfef2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/3357006aa19847d2a44a068b17c74258] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=47.3 K 2024-12-06T14:42:53,840 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93665f6451c14c2a8eca599e71698f5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496170578 2024-12-06T14:42:53,841 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1215914dcc6945ce9b12abda07220c23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733496171219 2024-12-06T14:42:53,841 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e67ae21a04407db33e3d628b4cfef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733496171950 2024-12-06T14:42:53,841 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3357006aa19847d2a44a068b17c74258, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733496172590 2024-12-06T14:42:53,848 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into 2fa0685c6b884efd8ca2b033f07160a5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:53,848 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:53,849 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=12, startTime=1733496173716; duration=0sec 2024-12-06T14:42:53,849 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:53,849 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:42:53,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742058_1234 (size=12151) 2024-12-06T14:42:53,863 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:53,863 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/2980861e962040f7af20a2e8d0e7b921 is 50, key is test_row_0/C:col10/1733496172590/Put/seqid=0 2024-12-06T14:42:53,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496173792, max timestamp=1733496173862Num uniques 859; 2024-12-06T14:42:53,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:53,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496173792, max timestamp=1733496173862Num uniques 859; 2024-12-06T14:42:53,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:53,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496173792, max timestamp=1733496173862Num uniques 859; 2024-12-06T14:42:53,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:53,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/6a0fd27d318a463fb23d24b982646645 2024-12-06T14:42:53,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/f5e159772d71419eb372cd796d22a501 is 50, key is test_row_0/B:col10/1733496173756/Put/seqid=0 2024-12-06T14:42:53,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742059_1235 (size=12425) 2024-12-06T14:42:53,916 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/2980861e962040f7af20a2e8d0e7b921 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2980861e962040f7af20a2e8d0e7b921 2024-12-06T14:42:53,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742060_1236 (size=12151) 2024-12-06T14:42:53,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:53,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:53,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:53,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:53,946 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into 2980861e962040f7af20a2e8d0e7b921(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:53,946 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:53,946 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=12, startTime=1733496173716; duration=0sec 2024-12-06T14:42:53,946 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:53,946 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:42:53,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1733496173900, max timestamp=1733496173928Num uniques 1018; 2024-12-06T14:42:53,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=7680, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-06T14:42:53,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7681, totalHeapSize=17712, min timestamp=1733496173900, max timestamp=1733496173928Num uniques 1018; 2024-12-06T14:42:53,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46611, new segment data size=7681, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-06T14:42:53,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7681, totalHeapSize=17712, min timestamp=1733496173900, max timestamp=1733496173928Num uniques 1018; 2024-12-06T14:42:53,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46611, new segment data size=7681, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-06T14:42:53,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733496173954, max timestamp=1733496173975Num uniques 1027; 2024-12-06T14:42:53,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-06T14:42:53,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733496173970, max timestamp=1733496173975Num uniques 1027; 2024-12-06T14:42:53,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-06T14:42:53,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733496173970, max timestamp=1733496173975Num uniques 1027; 2024-12-06T14:42:53,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-06T14:42:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496234004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496234005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496234006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733496173990, max timestamp=1733496174010Num uniques 1186; 2024-12-06T14:42:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-12-06T14:42:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733496173990, max timestamp=1733496174010Num uniques 1036; 2024-12-06T14:42:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-06T14:42:54,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733496174003, max timestamp=1733496174010Num uniques 1036; 2024-12-06T14:42:54,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-06T14:42:54,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733496174050, max timestamp=1733496174074Num uniques 1095; 2024-12-06T14:42:54,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-06T14:42:54,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733496174050, max timestamp=1733496174074Num uniques 1095; 2024-12-06T14:42:54,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-06T14:42:54,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733496174050, max timestamp=1733496174074Num uniques 1045; 2024-12-06T14:42:54,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-06T14:42:54,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:54,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:54,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:54,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1733496174116, max timestamp=1733496174123Num uniques 1104; 2024-12-06T14:42:54,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1733496174116, max timestamp=1733496174123Num uniques 1104; 2024-12-06T14:42:54,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=9334, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-06T14:42:54,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=9334, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-06T14:42:54,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496174116, max timestamp=1733496174123Num uniques 1104; 2024-12-06T14:42:54,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-12-06T14:42:54,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733496174139, max timestamp=1733496174148Num uniques 1063; 2024-12-06T14:42:54,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-06T14:42:54,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733496174139, max timestamp=1733496174148Num uniques 1063; 2024-12-06T14:42:54,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-06T14:42:54,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733496174139, max timestamp=1733496174148Num uniques 1009; 2024-12-06T14:42:54,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9748, suffix heap size=132904,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=213 2024-12-06T14:42:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496234173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496234173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496234174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496234177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733496174172, max timestamp=1733496174179Num uniques 1072; 2024-12-06T14:42:54,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-06T14:42:54,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733496174172, max timestamp=1733496174179Num uniques 1072; 2024-12-06T14:42:54,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-06T14:42:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733496174172, max timestamp=1733496174179Num uniques 1072; 2024-12-06T14:42:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-06T14:42:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733496174245, max timestamp=1733496174278Num uniques 1031; 2024-12-06T14:42:54,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-06T14:42:54,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733496174245, max timestamp=1733496174278Num uniques 1031; 2024-12-06T14:42:54,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-06T14:42:54,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733496174245, max timestamp=1733496174278Num uniques 1031; 2024-12-06T14:42:54,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-06T14:42:54,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733496174301, max timestamp=1733496174311Num uniques 1040; 2024-12-06T14:42:54,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-12-06T14:42:54,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733496174301, max timestamp=1733496174311Num uniques 1040; 2024-12-06T14:42:54,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-06T14:42:54,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733496174301, max timestamp=1733496174311Num uniques 1040; 2024-12-06T14:42:54,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-06T14:42:54,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/f5e159772d71419eb372cd796d22a501 2024-12-06T14:42:54,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/ff6fb81a9d884a27a03a2f71b541c899 is 50, key is test_row_0/C:col10/1733496173756/Put/seqid=0 2024-12-06T14:42:54,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742061_1237 (size=12151) 2024-12-06T14:42:54,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496234336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733496174335, max timestamp=1733496174340Num uniques 1199; 2024-12-06T14:42:54,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=9114, suffix heap size=152664,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=199 2024-12-06T14:42:54,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733496174335, max timestamp=1733496174340Num uniques 1049; 2024-12-06T14:42:54,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-06T14:42:54,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733496174335, max timestamp=1733496174340Num uniques 1049; 2024-12-06T14:42:54,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-06T14:42:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1733496174369, max timestamp=1733496174398Num uniques 1058; 2024-12-06T14:42:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-12-06T14:42:54,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1733496174369, max timestamp=1733496174398Num uniques 1058; 2024-12-06T14:42:54,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-12-06T14:42:54,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:54,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:54,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733496174369, max timestamp=1733496174398Num uniques 1058; 2024-12-06T14:42:54,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-06T14:42:54,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496234467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496234467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496234472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496234470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496234474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496234578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496234578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496234588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496234589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496234589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:54,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:54,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:54,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/ff6fb81a9d884a27a03a2f71b541c899 2024-12-06T14:42:54,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/6a0fd27d318a463fb23d24b982646645 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/6a0fd27d318a463fb23d24b982646645 2024-12-06T14:42:54,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/6a0fd27d318a463fb23d24b982646645, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T14:42:54,755 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-06T14:42:54,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/f5e159772d71419eb372cd796d22a501 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f5e159772d71419eb372cd796d22a501 2024-12-06T14:42:54,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f5e159772d71419eb372cd796d22a501, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T14:42:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/ff6fb81a9d884a27a03a2f71b541c899 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ff6fb81a9d884a27a03a2f71b541c899 2024-12-06T14:42:54,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ff6fb81a9d884a27a03a2f71b541c899, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T14:42:54,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=149.83 KB/153430 for 4828895d58f7749c326fda40500dce6a in 1012ms, sequenceid=169, compaction requested=false 2024-12-06T14:42:54,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:54,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733496174465, max timestamp=1733496174791Num uniques 1067; 2024-12-06T14:42:54,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-06T14:42:54,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733496174465, max timestamp=1733496174791Num uniques 1167; 2024-12-06T14:42:54,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733496174465, max timestamp=1733496174791Num uniques 1167; 2024-12-06T14:42:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:54,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=7642, suffix heap size=149336,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=167 2024-12-06T14:42:54,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=7642, suffix heap size=149336,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=167 2024-12-06T14:42:54,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=33.54 KB heapSize=85.41 KB 2024-12-06T14:42:54,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:54,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:54,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4abad3cc392642c197f01c267a5666a5 is 50, key is test_row_0/A:col10/1733496174791/Put/seqid=0 2024-12-06T14:42:54,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496174822, max timestamp=1733496174833Num uniques 859; 2024-12-06T14:42:54,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496174822, max timestamp=1733496174833Num uniques 859; 2024-12-06T14:42:54,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:54,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:54,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496174822, max timestamp=1733496174833Num uniques 859; 2024-12-06T14:42:54,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:42:54,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:54,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:54,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:54,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742062_1238 (size=12301) 2024-12-06T14:42:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496234863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733496174858, max timestamp=1733496174872Num uniques 1068; 2024-12-06T14:42:54,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T14:42:54,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733496174858, max timestamp=1733496174872Num uniques 1068; 2024-12-06T14:42:54,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T14:42:54,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733496174858, max timestamp=1733496174872Num uniques 1068; 2024-12-06T14:42:54,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T14:42:54,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733496174901, max timestamp=1733496174914Num uniques 1027; 2024-12-06T14:42:54,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-06T14:42:54,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10387, totalHeapSize=23848, min timestamp=1733496174901, max timestamp=1733496174915Num uniques 1027; 2024-12-06T14:42:54,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47027, new segment data size=10387, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-06T14:42:54,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733496174901, max timestamp=1733496174914Num uniques 1027; 2024-12-06T14:42:54,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-06T14:42:54,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496234958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496234960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496234961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496234963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:54,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496234978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496235069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496235078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496235078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496235078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:55,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496235188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4abad3cc392642c197f01c267a5666a5 2024-12-06T14:42:55,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496235278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496235288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496235298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496235298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3bc127dab8f44e869e6f3e76719c5a3d is 50, key is test_row_0/B:col10/1733496174791/Put/seqid=0 2024-12-06T14:42:55,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:55,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742063_1239 (size=12301) 2024-12-06T14:42:55,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3bc127dab8f44e869e6f3e76719c5a3d 2024-12-06T14:42:55,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/c8f82a7bc34949e9b42a49eb45ef5fe7 is 50, key is test_row_0/C:col10/1733496174791/Put/seqid=0 2024-12-06T14:42:55,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742064_1240 (size=12301) 2024-12-06T14:42:55,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/c8f82a7bc34949e9b42a49eb45ef5fe7 2024-12-06T14:42:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4abad3cc392642c197f01c267a5666a5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4abad3cc392642c197f01c267a5666a5 2024-12-06T14:42:55,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4abad3cc392642c197f01c267a5666a5, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T14:42:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3bc127dab8f44e869e6f3e76719c5a3d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3bc127dab8f44e869e6f3e76719c5a3d 2024-12-06T14:42:55,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3bc127dab8f44e869e6f3e76719c5a3d, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T14:42:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/c8f82a7bc34949e9b42a49eb45ef5fe7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c8f82a7bc34949e9b42a49eb45ef5fe7 2024-12-06T14:42:55,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c8f82a7bc34949e9b42a49eb45ef5fe7, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T14:42:55,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.20 KB/128208, currentSize=140.89 KB/144270 for 4828895d58f7749c326fda40500dce6a in 627ms, sequenceid=402, compaction requested=true 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:55,425 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:55,425 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:55,428 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:55,428 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:42:55,428 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,428 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/783e306ee19f4db7b2c48aa5362b77b8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/6a0fd27d318a463fb23d24b982646645, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4abad3cc392642c197f01c267a5666a5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.0 K 2024-12-06T14:42:55,429 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:55,429 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:42:55,430 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,430 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 783e306ee19f4db7b2c48aa5362b77b8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733496172590 2024-12-06T14:42:55,430 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/2fa0685c6b884efd8ca2b033f07160a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f5e159772d71419eb372cd796d22a501, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3bc127dab8f44e869e6f3e76719c5a3d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.0 K 2024-12-06T14:42:55,430 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa0685c6b884efd8ca2b033f07160a5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733496172590 2024-12-06T14:42:55,431 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a0fd27d318a463fb23d24b982646645, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733496173756 2024-12-06T14:42:55,431 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e159772d71419eb372cd796d22a501, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733496173756 2024-12-06T14:42:55,431 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 4abad3cc392642c197f01c267a5666a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733496174791 2024-12-06T14:42:55,432 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc127dab8f44e869e6f3e76719c5a3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733496174791 2024-12-06T14:42:55,442 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:55,443 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3cbe53d0d4cf49aba07746122cba0851 is 50, key is test_row_0/B:col10/1733496174791/Put/seqid=0 2024-12-06T14:42:55,456 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:55,456 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/550317f6a3334bdb82626942a5748cd3 is 50, key is test_row_0/A:col10/1733496174791/Put/seqid=0 2024-12-06T14:42:55,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742065_1241 (size=12677) 2024-12-06T14:42:55,474 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3cbe53d0d4cf49aba07746122cba0851 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3cbe53d0d4cf49aba07746122cba0851 2024-12-06T14:42:55,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T14:42:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,488 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=140.89 KB heapSize=398.53 KB 2024-12-06T14:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:55,538 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into 3cbe53d0d4cf49aba07746122cba0851(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:55,538 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:55,538 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=13, startTime=1733496175425; duration=0sec 2024-12-06T14:42:55,538 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:55,538 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:42:55,538 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:55,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742066_1242 (size=12677) 2024-12-06T14:42:55,540 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:55,541 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:42:55,541 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:55,541 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2980861e962040f7af20a2e8d0e7b921, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ff6fb81a9d884a27a03a2f71b541c899, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c8f82a7bc34949e9b42a49eb45ef5fe7] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.0 K 2024-12-06T14:42:55,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:55,544 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2980861e962040f7af20a2e8d0e7b921, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733496172590 2024-12-06T14:42:55,545 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff6fb81a9d884a27a03a2f71b541c899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733496173756 2024-12-06T14:42:55,545 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8f82a7bc34949e9b42a49eb45ef5fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733496174791 2024-12-06T14:42:55,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/92ec45283468459a8e70dadd97b21dca is 50, key is test_row_0/A:col10/1733496174961/Put/seqid=0 2024-12-06T14:42:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496235565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496235588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,601 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#204 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:55,602 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/61ebaa4023924fe8abcf34489f83dc29 is 50, key is test_row_0/C:col10/1733496174791/Put/seqid=0 2024-12-06T14:42:55,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496235599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496235608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496235608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742067_1243 (size=12301) 2024-12-06T14:42:55,617 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/92ec45283468459a8e70dadd97b21dca 2024-12-06T14:42:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:55,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742068_1244 (size=12677) 2024-12-06T14:42:55,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/7a8de30c7d614117a278c30b0c62f788 is 50, key is test_row_0/B:col10/1733496174961/Put/seqid=0 2024-12-06T14:42:55,641 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/61ebaa4023924fe8abcf34489f83dc29 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/61ebaa4023924fe8abcf34489f83dc29 2024-12-06T14:42:55,650 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into 61ebaa4023924fe8abcf34489f83dc29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:55,650 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:55,650 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=13, startTime=1733496175425; duration=0sec 2024-12-06T14:42:55,651 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:55,651 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:42:55,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742069_1245 (size=12301) 2024-12-06T14:42:55,658 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/7a8de30c7d614117a278c30b0c62f788 2024-12-06T14:42:55,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a147ac6e3d2c47f2a085d3f5f05a7f42 is 50, key is test_row_0/C:col10/1733496174961/Put/seqid=0 2024-12-06T14:42:55,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742070_1246 (size=12301) 2024-12-06T14:42:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496235678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496235888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:55,944 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/550317f6a3334bdb82626942a5748cd3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/550317f6a3334bdb82626942a5748cd3 2024-12-06T14:42:55,951 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 550317f6a3334bdb82626942a5748cd3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:55,951 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:55,951 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=13, startTime=1733496175425; duration=0sec 2024-12-06T14:42:55,952 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:55,952 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:42:56,071 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a147ac6e3d2c47f2a085d3f5f05a7f42 2024-12-06T14:42:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/92ec45283468459a8e70dadd97b21dca as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/92ec45283468459a8e70dadd97b21dca 2024-12-06T14:42:56,081 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/92ec45283468459a8e70dadd97b21dca, entries=150, sequenceid=474, filesize=12.0 K 2024-12-06T14:42:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/7a8de30c7d614117a278c30b0c62f788 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/7a8de30c7d614117a278c30b0c62f788 2024-12-06T14:42:56,088 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/7a8de30c7d614117a278c30b0c62f788, entries=150, sequenceid=474, filesize=12.0 K 2024-12-06T14:42:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a147ac6e3d2c47f2a085d3f5f05a7f42 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a147ac6e3d2c47f2a085d3f5f05a7f42 2024-12-06T14:42:56,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496236098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,102 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a147ac6e3d2c47f2a085d3f5f05a7f42, entries=150, sequenceid=474, filesize=12.0 K 2024-12-06T14:42:56,104 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~398.48 KB/408048, currentSize=40.25 KB/41220 for 4828895d58f7749c326fda40500dce6a in 616ms, sequenceid=474, compaction requested=false 2024-12-06T14:42:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-06T14:42:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-06T14:42:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:56,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-06T14:42:56,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-12-06T14:42:56,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:42:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:56,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ed842bb1fd2e465580afd0db371d208f is 50, key is test_row_0/A:col10/1733496175550/Put/seqid=0 2024-12-06T14:42:56,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6330 sec 2024-12-06T14:42:56,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742071_1247 (size=12301) 2024-12-06T14:42:56,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ed842bb1fd2e465580afd0db371d208f 2024-12-06T14:42:56,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/63866df553b841f49961671e8861b4a9 is 50, key is test_row_0/B:col10/1733496175550/Put/seqid=0 2024-12-06T14:42:56,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742072_1248 (size=12301) 2024-12-06T14:42:56,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496176140, max timestamp=1733496176148Num uniques 859; 2024-12-06T14:42:56,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:56,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496176140, max timestamp=1733496176148Num uniques 859; 2024-12-06T14:42:56,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:56,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496176140, max timestamp=1733496176148Num uniques 859; 2024-12-06T14:42:56,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496236174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496236176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496236179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496236199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496236288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496236288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496236289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496236498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496236499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496236499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/63866df553b841f49961671e8861b4a9 2024-12-06T14:42:56,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/05b2644b72d74d8cab83af6e87933ec2 is 50, key is test_row_0/C:col10/1733496175550/Put/seqid=0 2024-12-06T14:42:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742073_1249 (size=12301) 2024-12-06T14:42:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496236708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496236808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496236809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496236809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:56,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/05b2644b72d74d8cab83af6e87933ec2 2024-12-06T14:42:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ed842bb1fd2e465580afd0db371d208f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ed842bb1fd2e465580afd0db371d208f 2024-12-06T14:42:56,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ed842bb1fd2e465580afd0db371d208f, entries=150, sequenceid=486, filesize=12.0 K 2024-12-06T14:42:56,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/63866df553b841f49961671e8861b4a9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/63866df553b841f49961671e8861b4a9 2024-12-06T14:42:56,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/63866df553b841f49961671e8861b4a9, entries=150, sequenceid=486, filesize=12.0 K 2024-12-06T14:42:56,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/05b2644b72d74d8cab83af6e87933ec2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/05b2644b72d74d8cab83af6e87933ec2 2024-12-06T14:42:56,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/05b2644b72d74d8cab83af6e87933ec2, entries=150, sequenceid=486, filesize=12.0 K 2024-12-06T14:42:56,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4828895d58f7749c326fda40500dce6a in 873ms, sequenceid=486, compaction requested=true 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:56,983 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:56,983 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:56,984 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:56,984 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:42:56,984 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:56,984 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/550317f6a3334bdb82626942a5748cd3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/92ec45283468459a8e70dadd97b21dca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ed842bb1fd2e465580afd0db371d208f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.4 K 2024-12-06T14:42:56,985 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:56,985 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:42:56,985 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:56,985 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3cbe53d0d4cf49aba07746122cba0851, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/7a8de30c7d614117a278c30b0c62f788, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/63866df553b841f49961671e8861b4a9] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.4 K 2024-12-06T14:42:56,985 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550317f6a3334bdb82626942a5748cd3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733496174791 2024-12-06T14:42:56,985 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cbe53d0d4cf49aba07746122cba0851, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733496174791 2024-12-06T14:42:56,985 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92ec45283468459a8e70dadd97b21dca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733496174951 2024-12-06T14:42:56,986 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a8de30c7d614117a278c30b0c62f788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733496174951 2024-12-06T14:42:56,986 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed842bb1fd2e465580afd0db371d208f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733496175550 2024-12-06T14:42:56,987 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 63866df553b841f49961671e8861b4a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733496175550 2024-12-06T14:42:56,993 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:56,993 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd is 50, key is test_row_0/A:col10/1733496175550/Put/seqid=0 2024-12-06T14:42:56,998 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:56,998 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/bb1f248efa334559b7283143c9c555b1 is 50, key is test_row_0/B:col10/1733496175550/Put/seqid=0 2024-12-06T14:42:57,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742074_1250 (size=12779) 2024-12-06T14:42:57,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742075_1251 (size=12779) 2024-12-06T14:42:57,033 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd 2024-12-06T14:42:57,041 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 3e45e5d1c49a4fb5b09f8a8a7e19b0bd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:57,041 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:57,041 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=13, startTime=1733496176983; duration=0sec 2024-12-06T14:42:57,041 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:57,041 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:42:57,041 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:57,043 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:57,043 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:42:57,043 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:57,043 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/61ebaa4023924fe8abcf34489f83dc29, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a147ac6e3d2c47f2a085d3f5f05a7f42, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/05b2644b72d74d8cab83af6e87933ec2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.4 K 2024-12-06T14:42:57,044 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ebaa4023924fe8abcf34489f83dc29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733496174791 2024-12-06T14:42:57,044 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a147ac6e3d2c47f2a085d3f5f05a7f42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733496174951 2024-12-06T14:42:57,044 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b2644b72d74d8cab83af6e87933ec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733496175550 2024-12-06T14:42:57,069 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#212 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:57,070 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e6a5740652f9476e899d0aafdefae0a5 is 50, key is test_row_0/C:col10/1733496175550/Put/seqid=0 2024-12-06T14:42:57,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742076_1252 (size=12779) 2024-12-06T14:42:57,095 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e6a5740652f9476e899d0aafdefae0a5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e6a5740652f9476e899d0aafdefae0a5 2024-12-06T14:42:57,101 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into e6a5740652f9476e899d0aafdefae0a5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:57,101 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:57,101 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=13, startTime=1733496176983; duration=0sec 2024-12-06T14:42:57,101 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:57,101 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:42:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:57,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-06T14:42:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:42:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:42:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:42:57,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/28e6d7b8bc2a46498ad3c4caad036372 is 50, key is test_row_0/A:col10/1733496177110/Put/seqid=0 2024-12-06T14:42:57,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742077_1253 (size=14741) 2024-12-06T14:42:57,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/28e6d7b8bc2a46498ad3c4caad036372 2024-12-06T14:42:57,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e58045cf69cc49008e6b57e68f3a483e is 50, key is test_row_0/B:col10/1733496177110/Put/seqid=0 2024-12-06T14:42:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496237143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742078_1254 (size=12301) 2024-12-06T14:42:57,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496237258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496237318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496237318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496237318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,435 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/bb1f248efa334559b7283143c9c555b1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bb1f248efa334559b7283143c9c555b1 2024-12-06T14:42:57,440 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into bb1f248efa334559b7283143c9c555b1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:57,440 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:57,440 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=13, startTime=1733496176983; duration=0sec 2024-12-06T14:42:57,441 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:57,441 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:42:57,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496237468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e58045cf69cc49008e6b57e68f3a483e 2024-12-06T14:42:57,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/195164aa526f4730be8cefaeb30f0150 is 50, key is test_row_0/C:col10/1733496177110/Put/seqid=0 2024-12-06T14:42:57,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742079_1255 (size=12301) 2024-12-06T14:42:57,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/195164aa526f4730be8cefaeb30f0150 2024-12-06T14:42:57,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/28e6d7b8bc2a46498ad3c4caad036372 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28e6d7b8bc2a46498ad3c4caad036372 2024-12-06T14:42:57,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28e6d7b8bc2a46498ad3c4caad036372, entries=200, sequenceid=526, filesize=14.4 K 2024-12-06T14:42:57,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e58045cf69cc49008e6b57e68f3a483e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e58045cf69cc49008e6b57e68f3a483e 2024-12-06T14:42:57,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e58045cf69cc49008e6b57e68f3a483e, entries=150, sequenceid=526, filesize=12.0 K 2024-12-06T14:42:57,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/195164aa526f4730be8cefaeb30f0150 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/195164aa526f4730be8cefaeb30f0150 2024-12-06T14:42:57,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/195164aa526f4730be8cefaeb30f0150, entries=150, sequenceid=526, filesize=12.0 K 2024-12-06T14:42:57,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 4828895d58f7749c326fda40500dce6a in 481ms, sequenceid=526, compaction requested=false 2024-12-06T14:42:57,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T14:42:57,638 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:57,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-06T14:42:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T14:42:57,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:57,642 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:57,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:57,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:42:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:57,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4d6f41e677454afb89497b7278591ab3 is 50, key is test_row_0/A:col10/1733496177141/Put/seqid=0 2024-12-06T14:42:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T14:42:57,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742080_1256 (size=12301) 2024-12-06T14:42:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496177777, max timestamp=1733496177787Num uniques 859; 2024-12-06T14:42:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496177777, max timestamp=1733496177787Num uniques 859; 2024-12-06T14:42:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496177777, max timestamp=1733496177787Num uniques 859; 2024-12-06T14:42:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:42:57,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:57,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:57,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496237832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496237836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496237938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:57,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:57,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:57,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:57,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:57,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496237952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T14:42:58,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496238149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496238158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4d6f41e677454afb89497b7278591ab3 2024-12-06T14:42:58,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/aa3aa6e7ce90442498fefe05fad003fe is 50, key is test_row_0/B:col10/1733496177141/Put/seqid=0 2024-12-06T14:42:58,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742081_1257 (size=12301) 2024-12-06T14:42:58,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/aa3aa6e7ce90442498fefe05fad003fe 2024-12-06T14:42:58,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e9a9fc678d574042b680c22cd6393087 is 50, key is test_row_0/C:col10/1733496177141/Put/seqid=0 2024-12-06T14:42:58,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742082_1258 (size=12301) 2024-12-06T14:42:58,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T14:42:58,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496238328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496238328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,331 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,331 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496238338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,340 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:42:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496238459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496238468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:42:58,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e9a9fc678d574042b680c22cd6393087 2024-12-06T14:42:58,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4d6f41e677454afb89497b7278591ab3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4d6f41e677454afb89497b7278591ab3 2024-12-06T14:42:58,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4d6f41e677454afb89497b7278591ab3, entries=150, sequenceid=537, filesize=12.0 K 2024-12-06T14:42:58,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/aa3aa6e7ce90442498fefe05fad003fe as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/aa3aa6e7ce90442498fefe05fad003fe 2024-12-06T14:42:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/aa3aa6e7ce90442498fefe05fad003fe, entries=150, sequenceid=537, filesize=12.0 K 2024-12-06T14:42:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/e9a9fc678d574042b680c22cd6393087 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e9a9fc678d574042b680c22cd6393087 2024-12-06T14:42:58,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e9a9fc678d574042b680c22cd6393087, entries=150, sequenceid=537, filesize=12.0 K 2024-12-06T14:42:58,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4828895d58f7749c326fda40500dce6a in 915ms, sequenceid=537, compaction requested=true 2024-12-06T14:42:58,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:42:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:42:58,637 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:42:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T14:42:58,637 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:58,638 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:58,638 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:42:58,638 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,638 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28e6d7b8bc2a46498ad3c4caad036372, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4d6f41e677454afb89497b7278591ab3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=38.9 K 2024-12-06T14:42:58,639 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:58,639 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:42:58,639 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,639 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e45e5d1c49a4fb5b09f8a8a7e19b0bd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733496175550 2024-12-06T14:42:58,639 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bb1f248efa334559b7283143c9c555b1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e58045cf69cc49008e6b57e68f3a483e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/aa3aa6e7ce90442498fefe05fad003fe] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.5 K 2024-12-06T14:42:58,640 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 28e6d7b8bc2a46498ad3c4caad036372, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733496176173 2024-12-06T14:42:58,640 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1f248efa334559b7283143c9c555b1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733496175550 2024-12-06T14:42:58,640 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d6f41e677454afb89497b7278591ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733496177130 2024-12-06T14:42:58,640 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58045cf69cc49008e6b57e68f3a483e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733496176173 2024-12-06T14:42:58,641 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa3aa6e7ce90442498fefe05fad003fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733496177130 2024-12-06T14:42:58,675 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:58,675 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/1024fb763aa24af4aedb6e4a71158224 is 50, key is test_row_0/B:col10/1733496177141/Put/seqid=0 2024-12-06T14:42:58,682 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:58,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/7d568fd7b54b474dac4393c83229fd83 is 50, key is test_row_0/A:col10/1733496177141/Put/seqid=0 2024-12-06T14:42:58,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T14:42:58,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,716 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-06T14:42:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:42:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742084_1260 (size=12881) 2024-12-06T14:42:58,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742083_1259 (size=12881) 2024-12-06T14:42:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ad3317d2aee1482cab256dbf05d9cf17 is 50, key is test_row_0/A:col10/1733496177830/Put/seqid=0 2024-12-06T14:42:58,728 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/7d568fd7b54b474dac4393c83229fd83 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/7d568fd7b54b474dac4393c83229fd83 2024-12-06T14:42:58,735 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 7d568fd7b54b474dac4393c83229fd83(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:58,735 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:58,735 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=13, startTime=1733496178637; duration=0sec 2024-12-06T14:42:58,735 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:42:58,735 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:42:58,735 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:42:58,737 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:42:58,737 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:42:58,737 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:58,737 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e6a5740652f9476e899d0aafdefae0a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/195164aa526f4730be8cefaeb30f0150, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e9a9fc678d574042b680c22cd6393087] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.5 K 2024-12-06T14:42:58,737 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a5740652f9476e899d0aafdefae0a5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733496175550 2024-12-06T14:42:58,738 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 195164aa526f4730be8cefaeb30f0150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733496176173 2024-12-06T14:42:58,739 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a9fc678d574042b680c22cd6393087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733496177130 2024-12-06T14:42:58,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742085_1261 (size=12301) 2024-12-06T14:42:58,747 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:42:58,747 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1d836000a9c04c118880e37ca7d36798 is 50, key is test_row_0/C:col10/1733496177141/Put/seqid=0 2024-12-06T14:42:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742086_1262 (size=12881) 2024-12-06T14:42:58,769 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/1d836000a9c04c118880e37ca7d36798 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1d836000a9c04c118880e37ca7d36798 2024-12-06T14:42:58,777 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into 1d836000a9c04c118880e37ca7d36798(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:58,777 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:58,777 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=13, startTime=1733496178637; duration=0sec 2024-12-06T14:42:58,777 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:58,777 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:42:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T14:42:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:42:58,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:42:58,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496238986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:58,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496238987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496239099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496239099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,129 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/1024fb763aa24af4aedb6e4a71158224 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1024fb763aa24af4aedb6e4a71158224 2024-12-06T14:42:59,136 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into 1024fb763aa24af4aedb6e4a71158224(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:42:59,136 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:59,136 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=13, startTime=1733496178637; duration=0sec 2024-12-06T14:42:59,137 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:42:59,137 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:42:59,142 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ad3317d2aee1482cab256dbf05d9cf17 2024-12-06T14:42:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/dffa18f6c144475b9d590507e39925ac is 50, key is test_row_0/B:col10/1733496177830/Put/seqid=0 2024-12-06T14:42:59,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742087_1263 (size=12301) 2024-12-06T14:42:59,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496239309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:59,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496239309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,611 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/dffa18f6c144475b9d590507e39925ac 2024-12-06T14:42:59,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496239619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:42:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496239619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:42:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:42:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:42:59,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/879045276cb24f99a58c7196ee58072b is 50, key is test_row_0/C:col10/1733496177830/Put/seqid=0 2024-12-06T14:42:59,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742088_1264 (size=12301) 2024-12-06T14:42:59,673 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/879045276cb24f99a58c7196ee58072b 2024-12-06T14:42:59,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ad3317d2aee1482cab256dbf05d9cf17 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ad3317d2aee1482cab256dbf05d9cf17 2024-12-06T14:42:59,694 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ad3317d2aee1482cab256dbf05d9cf17, entries=150, sequenceid=574, filesize=12.0 K 2024-12-06T14:42:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/dffa18f6c144475b9d590507e39925ac as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/dffa18f6c144475b9d590507e39925ac 2024-12-06T14:42:59,702 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/dffa18f6c144475b9d590507e39925ac, entries=150, sequenceid=574, filesize=12.0 K 2024-12-06T14:42:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/879045276cb24f99a58c7196ee58072b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/879045276cb24f99a58c7196ee58072b 2024-12-06T14:42:59,718 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/879045276cb24f99a58c7196ee58072b, entries=150, sequenceid=574, filesize=12.0 K 2024-12-06T14:42:59,719 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4828895d58f7749c326fda40500dce6a in 1003ms, sequenceid=574, compaction requested=false 2024-12-06T14:42:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:42:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-06T14:42:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-06T14:42:59,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-06T14:42:59,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0790 sec 2024-12-06T14:42:59,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0920 sec 2024-12-06T14:42:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T14:42:59,788 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:42:59,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:42:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-06T14:42:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T14:42:59,793 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:42:59,794 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:42:59,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:42:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T14:42:59,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:42:59,947 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:42:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:42:59,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/b40abb48e14948959bc7abf9766bacfc is 50, key is test_row_0/A:col10/1733496178986/Put/seqid=0 2024-12-06T14:43:00,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742089_1265 (size=12301) 2024-12-06T14:43:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T14:43:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:00,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496180165, max timestamp=1733496180180Num uniques 859; 2024-12-06T14:43:00,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496180165, max timestamp=1733496180180Num uniques 859; 2024-12-06T14:43:00,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:43:00,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:43:00,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496180165, max timestamp=1733496180180Num uniques 859; 2024-12-06T14:43:00,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:43:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496240230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496240231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496240349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496240349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,352 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:00,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496240349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496240349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,353 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496240359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,361 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T14:43:00,431 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/b40abb48e14948959bc7abf9766bacfc 2024-12-06T14:43:00,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/1f495db7bdd64477b173f81e7dbae164 is 50, key is test_row_0/B:col10/1733496178986/Put/seqid=0 2024-12-06T14:43:00,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742090_1266 (size=12301) 2024-12-06T14:43:00,539 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/1f495db7bdd64477b173f81e7dbae164 2024-12-06T14:43:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/dbfe166ec9814393a2f5c87cc167fe05 is 50, key is test_row_0/C:col10/1733496178986/Put/seqid=0 2024-12-06T14:43:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496240558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496240558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742091_1267 (size=12301) 2024-12-06T14:43:00,583 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/dbfe166ec9814393a2f5c87cc167fe05 2024-12-06T14:43:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/b40abb48e14948959bc7abf9766bacfc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b40abb48e14948959bc7abf9766bacfc 2024-12-06T14:43:00,603 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b40abb48e14948959bc7abf9766bacfc, entries=150, sequenceid=587, filesize=12.0 K 2024-12-06T14:43:00,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/1f495db7bdd64477b173f81e7dbae164 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1f495db7bdd64477b173f81e7dbae164 2024-12-06T14:43:00,613 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1f495db7bdd64477b173f81e7dbae164, entries=150, sequenceid=587, filesize=12.0 K 2024-12-06T14:43:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/dbfe166ec9814393a2f5c87cc167fe05 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/dbfe166ec9814393a2f5c87cc167fe05 2024-12-06T14:43:00,622 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/dbfe166ec9814393a2f5c87cc167fe05, entries=150, sequenceid=587, filesize=12.0 K 2024-12-06T14:43:00,623 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4828895d58f7749c326fda40500dce6a in 676ms, sequenceid=587, compaction requested=true 2024-12-06T14:43:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-06T14:43:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-06T14:43:00,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-06T14:43:00,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 831 msec 2024-12-06T14:43:00,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 839 msec 2024-12-06T14:43:00,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-06T14:43:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:00,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:00,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:43:00,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:00,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:43:00,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:00,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:43:00,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/28a8a83f846942c1887872d15ce72a71 is 50, key is test_row_0/A:col10/1733496180228/Put/seqid=0 2024-12-06T14:43:00,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742092_1268 (size=17181) 2024-12-06T14:43:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496240901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/28a8a83f846942c1887872d15ce72a71 2024-12-06T14:43:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496240901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:00,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/cd3e3f2555024bf891b7986ac9613440 is 50, key is test_row_0/B:col10/1733496180228/Put/seqid=0 2024-12-06T14:43:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T14:43:00,929 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:00,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-06T14:43:00,932 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:00,933 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:00,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:00,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742093_1269 (size=12301) 2024-12-06T14:43:00,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/cd3e3f2555024bf891b7986ac9613440 2024-12-06T14:43:01,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/ee6dbf7213064bc699264928d96b8bf3 is 50, key is test_row_0/C:col10/1733496180228/Put/seqid=0 2024-12-06T14:43:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496241018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496241018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:01,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742094_1270 (size=12301) 2024-12-06T14:43:01,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496241228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496241228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:01,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/ee6dbf7213064bc699264928d96b8bf3 2024-12-06T14:43:01,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/28a8a83f846942c1887872d15ce72a71 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28a8a83f846942c1887872d15ce72a71 2024-12-06T14:43:01,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28a8a83f846942c1887872d15ce72a71, entries=250, sequenceid=625, filesize=16.8 K 2024-12-06T14:43:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/cd3e3f2555024bf891b7986ac9613440 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/cd3e3f2555024bf891b7986ac9613440 2024-12-06T14:43:01,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/cd3e3f2555024bf891b7986ac9613440, entries=150, sequenceid=625, filesize=12.0 K 2024-12-06T14:43:01,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/ee6dbf7213064bc699264928d96b8bf3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ee6dbf7213064bc699264928d96b8bf3 2024-12-06T14:43:01,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ee6dbf7213064bc699264928d96b8bf3, entries=150, sequenceid=625, filesize=12.0 K 2024-12-06T14:43:01,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=40.25 KB/41220 for 4828895d58f7749c326fda40500dce6a in 610ms, sequenceid=625, compaction requested=true 2024-12-06T14:43:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:01,484 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:01,484 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:01,486 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:01,486 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:01,486 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:43:01,486 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:43:01,486 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,486 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,486 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1024fb763aa24af4aedb6e4a71158224, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/dffa18f6c144475b9d590507e39925ac, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1f495db7bdd64477b173f81e7dbae164, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/cd3e3f2555024bf891b7986ac9613440] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=48.6 K 2024-12-06T14:43:01,486 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/7d568fd7b54b474dac4393c83229fd83, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ad3317d2aee1482cab256dbf05d9cf17, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b40abb48e14948959bc7abf9766bacfc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28a8a83f846942c1887872d15ce72a71] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=53.4 K 2024-12-06T14:43:01,486 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d568fd7b54b474dac4393c83229fd83, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733496177130 2024-12-06T14:43:01,486 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1024fb763aa24af4aedb6e4a71158224, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733496177130 2024-12-06T14:43:01,487 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad3317d2aee1482cab256dbf05d9cf17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733496177824 2024-12-06T14:43:01,487 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting dffa18f6c144475b9d590507e39925ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733496177824 2024-12-06T14:43:01,487 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40abb48e14948959bc7abf9766bacfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733496178973 2024-12-06T14:43:01,487 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f495db7bdd64477b173f81e7dbae164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733496178973 2024-12-06T14:43:01,487 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a8a83f846942c1887872d15ce72a71, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733496180197 2024-12-06T14:43:01,488 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3e3f2555024bf891b7986ac9613440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733496180228 2024-12-06T14:43:01,511 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:01,511 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/b702c8d4383947458749da927deba369 is 50, key is test_row_0/A:col10/1733496180228/Put/seqid=0 2024-12-06T14:43:01,523 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:01,527 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/eadef417db47422a900df0ce6ff7604d is 50, key is test_row_0/B:col10/1733496180228/Put/seqid=0 2024-12-06T14:43:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:01,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:01,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742095_1271 (size=13017) 2024-12-06T14:43:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742096_1272 (size=13017) 2024-12-06T14:43:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:01,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/9406f5902d124c5b8355b6ab3d0db1fa is 50, key is test_row_0/A:col10/1733496181542/Put/seqid=0 2024-12-06T14:43:01,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742097_1273 (size=14741) 2024-12-06T14:43:01,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496181580, max timestamp=1733496181598Num uniques 859; 2024-12-06T14:43:01,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:43:01,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496181580, max timestamp=1733496181598Num uniques 859; 2024-12-06T14:43:01,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:43:01,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496181580, max timestamp=1733496181598Num uniques 859; 2024-12-06T14:43:01,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:43:01,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496241660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496241661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496241778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496241778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:01,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:01,954 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/b702c8d4383947458749da927deba369 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b702c8d4383947458749da927deba369 2024-12-06T14:43:01,958 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into b702c8d4383947458749da927deba369(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:01,958 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:01,959 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=12, startTime=1733496181484; duration=0sec 2024-12-06T14:43:01,959 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:01,959 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:43:01,959 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:01,960 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:01,960 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:43:01,960 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:01,960 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1d836000a9c04c118880e37ca7d36798, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/879045276cb24f99a58c7196ee58072b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/dbfe166ec9814393a2f5c87cc167fe05, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ee6dbf7213064bc699264928d96b8bf3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=48.6 K 2024-12-06T14:43:01,960 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d836000a9c04c118880e37ca7d36798, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733496177130 2024-12-06T14:43:01,961 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879045276cb24f99a58c7196ee58072b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733496177824 2024-12-06T14:43:01,961 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbfe166ec9814393a2f5c87cc167fe05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733496178973 2024-12-06T14:43:01,962 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6dbf7213064bc699264928d96b8bf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733496180228 2024-12-06T14:43:01,962 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/eadef417db47422a900df0ce6ff7604d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/eadef417db47422a900df0ce6ff7604d 2024-12-06T14:43:01,968 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into eadef417db47422a900df0ce6ff7604d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:01,968 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:01,968 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=12, startTime=1733496181484; duration=0sec 2024-12-06T14:43:01,968 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:01,968 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:43:01,972 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:01,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a911f2bc2c3744a7b7a4e88e04b20172 is 50, key is test_row_0/C:col10/1733496180228/Put/seqid=0 2024-12-06T14:43:01,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/9406f5902d124c5b8355b6ab3d0db1fa 2024-12-06T14:43:01,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/14defd1a3a0641bf942cd99ebaff48c0 is 50, key is test_row_0/B:col10/1733496181542/Put/seqid=0 2024-12-06T14:43:01,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742098_1274 (size=13017) 2024-12-06T14:43:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496241988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,993 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/a911f2bc2c3744a7b7a4e88e04b20172 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a911f2bc2c3744a7b7a4e88e04b20172 2024-12-06T14:43:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496241992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:01,997 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into a911f2bc2c3744a7b7a4e88e04b20172(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:01,997 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:01,997 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=12, startTime=1733496181485; duration=0sec 2024-12-06T14:43:01,997 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:01,997 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:43:01,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742099_1275 (size=12301) 2024-12-06T14:43:02,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:02,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:02,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496242298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:02,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496242298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:02,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/14defd1a3a0641bf942cd99ebaff48c0 2024-12-06T14:43:02,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/0c80442cbf7249f9a9dc283f78590943 is 50, key is test_row_0/C:col10/1733496181542/Put/seqid=0 2024-12-06T14:43:02,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742100_1276 (size=12301) 2024-12-06T14:43:02,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:02,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:02,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496242808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496242808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:02,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:02,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:02,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:02,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/0c80442cbf7249f9a9dc283f78590943 2024-12-06T14:43:02,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/9406f5902d124c5b8355b6ab3d0db1fa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9406f5902d124c5b8355b6ab3d0db1fa 2024-12-06T14:43:02,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9406f5902d124c5b8355b6ab3d0db1fa, entries=200, sequenceid=636, filesize=14.4 K 2024-12-06T14:43:02,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/14defd1a3a0641bf942cd99ebaff48c0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14defd1a3a0641bf942cd99ebaff48c0 2024-12-06T14:43:02,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14defd1a3a0641bf942cd99ebaff48c0, entries=150, sequenceid=636, filesize=12.0 K 2024-12-06T14:43:02,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/0c80442cbf7249f9a9dc283f78590943 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/0c80442cbf7249f9a9dc283f78590943 2024-12-06T14:43:02,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/0c80442cbf7249f9a9dc283f78590943, entries=150, sequenceid=636, filesize=12.0 K 2024-12-06T14:43:02,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4828895d58f7749c326fda40500dce6a in 1299ms, sequenceid=636, compaction requested=false 2024-12-06T14:43:02,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:02,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T14:43:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:02,934 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-06T14:43:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:02,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ca6d658189534e0a831bc4d5b0698a38 is 50, key is test_row_0/A:col10/1733496181636/Put/seqid=0 2024-12-06T14:43:02,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742101_1277 (size=12301) 2024-12-06T14:43:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:03,344 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ca6d658189534e0a831bc4d5b0698a38 2024-12-06T14:43:03,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e2f8f0c8d8184bae8750346f4967ded6 is 50, key is test_row_0/B:col10/1733496181636/Put/seqid=0 2024-12-06T14:43:03,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742102_1278 (size=12301) 2024-12-06T14:43:03,756 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e2f8f0c8d8184bae8750346f4967ded6 2024-12-06T14:43:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/07eafe66f0394d0a9d109fd4e5483a5b is 50, key is test_row_0/C:col10/1733496181636/Put/seqid=0 2024-12-06T14:43:03,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742103_1279 (size=12301) 2024-12-06T14:43:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:03,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496243841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496243841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496243948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:03,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496243958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496244158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,167 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/07eafe66f0394d0a9d109fd4e5483a5b 2024-12-06T14:43:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/ca6d658189534e0a831bc4d5b0698a38 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ca6d658189534e0a831bc4d5b0698a38 2024-12-06T14:43:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496244168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,174 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ca6d658189534e0a831bc4d5b0698a38, entries=150, sequenceid=674, filesize=12.0 K 2024-12-06T14:43:04,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/e2f8f0c8d8184bae8750346f4967ded6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e2f8f0c8d8184bae8750346f4967ded6 2024-12-06T14:43:04,184 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e2f8f0c8d8184bae8750346f4967ded6, entries=150, sequenceid=674, filesize=12.0 K 2024-12-06T14:43:04,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/07eafe66f0394d0a9d109fd4e5483a5b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/07eafe66f0394d0a9d109fd4e5483a5b 2024-12-06T14:43:04,190 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/07eafe66f0394d0a9d109fd4e5483a5b, entries=150, sequenceid=674, filesize=12.0 K 2024-12-06T14:43:04,191 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 4828895d58f7749c326fda40500dce6a in 1256ms, sequenceid=674, compaction requested=true 2024-12-06T14:43:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-06T14:43:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-06T14:43:04,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-06T14:43:04,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2590 sec 2024-12-06T14:43:04,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 3.2660 sec 2024-12-06T14:43:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:04,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T14:43:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:04,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/85ef1f5e334949b1ab3d096ce306e6d8 is 50, key is test_row_0/A:col10/1733496183839/Put/seqid=0 2024-12-06T14:43:04,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742104_1280 (size=12301) 2024-12-06T14:43:04,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496244414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496244415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496244418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496244468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496244478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496244528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496244528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496244528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496244738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496244738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496244738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/85ef1f5e334949b1ab3d096ce306e6d8 2024-12-06T14:43:04,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/8d9d968e989747cba66b713f9dc21a59 is 50, key is test_row_0/B:col10/1733496183839/Put/seqid=0 2024-12-06T14:43:04,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742105_1281 (size=12301) 2024-12-06T14:43:04,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496244978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496244988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496245048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496245048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496245048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T14:43:05,099 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:05,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-06T14:43:05,101 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T14:43:05,102 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:05,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:05,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/8d9d968e989747cba66b713f9dc21a59 2024-12-06T14:43:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T14:43:05,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/435210e422dd4ec6a6213aaa1b73f787 is 50, key is test_row_0/C:col10/1733496183839/Put/seqid=0 2024-12-06T14:43:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742106_1282 (size=12301) 2024-12-06T14:43:05,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T14:43:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T14:43:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T14:43:05,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T14:43:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496245558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496245558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496245558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:05,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/435210e422dd4ec6a6213aaa1b73f787 2024-12-06T14:43:05,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/85ef1f5e334949b1ab3d096ce306e6d8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/85ef1f5e334949b1ab3d096ce306e6d8 2024-12-06T14:43:05,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/85ef1f5e334949b1ab3d096ce306e6d8, entries=150, sequenceid=688, filesize=12.0 K 2024-12-06T14:43:05,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/8d9d968e989747cba66b713f9dc21a59 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/8d9d968e989747cba66b713f9dc21a59 2024-12-06T14:43:05,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/8d9d968e989747cba66b713f9dc21a59, entries=150, sequenceid=688, filesize=12.0 K 2024-12-06T14:43:05,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/435210e422dd4ec6a6213aaa1b73f787 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/435210e422dd4ec6a6213aaa1b73f787 2024-12-06T14:43:05,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/435210e422dd4ec6a6213aaa1b73f787, entries=150, sequenceid=688, filesize=12.0 K 2024-12-06T14:43:05,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4828895d58f7749c326fda40500dce6a in 1269ms, sequenceid=688, compaction requested=true 2024-12-06T14:43:05,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:05,651 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:05,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:05,652 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:05,653 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:05,653 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:05,653 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:43:05,653 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:43:05,653 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,653 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,653 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/eadef417db47422a900df0ce6ff7604d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14defd1a3a0641bf942cd99ebaff48c0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e2f8f0c8d8184bae8750346f4967ded6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/8d9d968e989747cba66b713f9dc21a59] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=48.8 K 2024-12-06T14:43:05,653 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b702c8d4383947458749da927deba369, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9406f5902d124c5b8355b6ab3d0db1fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ca6d658189534e0a831bc4d5b0698a38, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/85ef1f5e334949b1ab3d096ce306e6d8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=51.1 K 2024-12-06T14:43:05,654 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting eadef417db47422a900df0ce6ff7604d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733496180228 2024-12-06T14:43:05,654 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting b702c8d4383947458749da927deba369, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733496180228 2024-12-06T14:43:05,655 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 14defd1a3a0641bf942cd99ebaff48c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733496180888 2024-12-06T14:43:05,655 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9406f5902d124c5b8355b6ab3d0db1fa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733496180888 2024-12-06T14:43:05,655 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f8f0c8d8184bae8750346f4967ded6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1733496181636 2024-12-06T14:43:05,655 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6d658189534e0a831bc4d5b0698a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1733496181636 2024-12-06T14:43:05,656 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d9d968e989747cba66b713f9dc21a59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733496183833 2024-12-06T14:43:05,656 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85ef1f5e334949b1ab3d096ce306e6d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733496183833 2024-12-06T14:43:05,669 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:05,670 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/bc9db2bfc83443779529edc14db33c56 is 50, key is test_row_0/B:col10/1733496183839/Put/seqid=0 2024-12-06T14:43:05,674 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:05,674 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/756deb9f7d2047b4a39ddf624a5107f9 is 50, key is test_row_0/A:col10/1733496183839/Put/seqid=0 2024-12-06T14:43:05,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742107_1283 (size=13153) 2024-12-06T14:43:05,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742108_1284 (size=13153) 2024-12-06T14:43:05,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T14:43:05,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:05,712 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:43:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/8e2787cdd278413e9d3880389be17272 is 50, key is test_row_0/A:col10/1733496184406/Put/seqid=0 2024-12-06T14:43:05,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742109_1285 (size=12301) 2024-12-06T14:43:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T14:43:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:05,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496246025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496246029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,096 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/bc9db2bfc83443779529edc14db33c56 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bc9db2bfc83443779529edc14db33c56 2024-12-06T14:43:06,099 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/756deb9f7d2047b4a39ddf624a5107f9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/756deb9f7d2047b4a39ddf624a5107f9 2024-12-06T14:43:06,103 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into bc9db2bfc83443779529edc14db33c56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:06,103 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:06,103 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=12, startTime=1733496185652; duration=0sec 2024-12-06T14:43:06,103 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:06,105 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:43:06,105 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:06,107 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 756deb9f7d2047b4a39ddf624a5107f9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:06,107 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:06,107 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=12, startTime=1733496185651; duration=0sec 2024-12-06T14:43:06,107 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:06,107 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:43:06,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:06,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:43:06,108 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:06,108 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a911f2bc2c3744a7b7a4e88e04b20172, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/0c80442cbf7249f9a9dc283f78590943, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/07eafe66f0394d0a9d109fd4e5483a5b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/435210e422dd4ec6a6213aaa1b73f787] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=48.8 K 2024-12-06T14:43:06,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a911f2bc2c3744a7b7a4e88e04b20172, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733496180228 2024-12-06T14:43:06,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c80442cbf7249f9a9dc283f78590943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733496180888 2024-12-06T14:43:06,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 07eafe66f0394d0a9d109fd4e5483a5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1733496181636 2024-12-06T14:43:06,110 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 435210e422dd4ec6a6213aaa1b73f787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733496183833 2024-12-06T14:43:06,122 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/8e2787cdd278413e9d3880389be17272 2024-12-06T14:43:06,127 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:06,127 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/9a286bb5b488439499daddb0c6adb4b2 is 50, key is test_row_0/C:col10/1733496183839/Put/seqid=0 2024-12-06T14:43:06,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3ebd2ea7e6f54aa38e43e813de4e7127 is 50, key is test_row_0/B:col10/1733496184406/Put/seqid=0 2024-12-06T14:43:06,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742110_1286 (size=13153) 2024-12-06T14:43:06,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742111_1287 (size=12301) 2024-12-06T14:43:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496246138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496246138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T14:43:06,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496246348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496246348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,538 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3ebd2ea7e6f54aa38e43e813de4e7127 2024-12-06T14:43:06,539 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/9a286bb5b488439499daddb0c6adb4b2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/9a286bb5b488439499daddb0c6adb4b2 2024-12-06T14:43:06,546 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into 9a286bb5b488439499daddb0c6adb4b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:06,546 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:06,546 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=12, startTime=1733496185652; duration=0sec 2024-12-06T14:43:06,546 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:06,546 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:43:06,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/60e646e5804f4b24b98abefa50b03383 is 50, key is test_row_0/C:col10/1733496184406/Put/seqid=0 2024-12-06T14:43:06,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742112_1288 (size=12301) 2024-12-06T14:43:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496246568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496246569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,570 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,571 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496246578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,580 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 4828895d58f7749c326fda40500dce6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496246658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496246659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:06,951 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/60e646e5804f4b24b98abefa50b03383 2024-12-06T14:43:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/8e2787cdd278413e9d3880389be17272 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/8e2787cdd278413e9d3880389be17272 2024-12-06T14:43:06,959 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/8e2787cdd278413e9d3880389be17272, entries=150, sequenceid=706, filesize=12.0 K 2024-12-06T14:43:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/3ebd2ea7e6f54aa38e43e813de4e7127 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3ebd2ea7e6f54aa38e43e813de4e7127 2024-12-06T14:43:06,965 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3ebd2ea7e6f54aa38e43e813de4e7127, entries=150, sequenceid=706, filesize=12.0 K 2024-12-06T14:43:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/60e646e5804f4b24b98abefa50b03383 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/60e646e5804f4b24b98abefa50b03383 2024-12-06T14:43:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/60e646e5804f4b24b98abefa50b03383, entries=150, sequenceid=706, filesize=12.0 K 2024-12-06T14:43:06,970 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4828895d58f7749c326fda40500dce6a in 1258ms, sequenceid=706, compaction requested=false 2024-12-06T14:43:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-06T14:43:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-06T14:43:06,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-06T14:43:06,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-06T14:43:06,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.8730 sec 2024-12-06T14:43:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:07,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:43:07,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:07,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/708fe60b91d64e81bf5d15cb18eaf719 is 50, key is test_row_0/A:col10/1733496186016/Put/seqid=0 2024-12-06T14:43:07,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742113_1289 (size=14741) 2024-12-06T14:43:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496247221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496247221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T14:43:07,253 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:07,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-06T14:43:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T14:43:07,255 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:07,256 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:07,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:07,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496247329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496247329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T14:43:07,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:07,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:07,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496247538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496247538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:07,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:07,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T14:43:07,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/708fe60b91d64e81bf5d15cb18eaf719 2024-12-06T14:43:07,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/b6f9153d57274479980299cdcd13b756 is 50, key is test_row_0/B:col10/1733496186016/Put/seqid=0 2024-12-06T14:43:07,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742114_1290 (size=12301) 2024-12-06T14:43:07,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496247848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496247848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:07,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:07,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:07,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:07,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T14:43:08,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/b6f9153d57274479980299cdcd13b756 2024-12-06T14:43:08,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:08,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:08,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/770885aed34e4d168f6179721ed9fae7 is 50, key is test_row_0/C:col10/1733496186016/Put/seqid=0 2024-12-06T14:43:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742115_1291 (size=12301) 2024-12-06T14:43:08,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38606 deadline: 1733496248358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1733496248358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T14:43:08,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/770885aed34e4d168f6179721ed9fae7 2024-12-06T14:43:08,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/708fe60b91d64e81bf5d15cb18eaf719 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/708fe60b91d64e81bf5d15cb18eaf719 2024-12-06T14:43:08,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/708fe60b91d64e81bf5d15cb18eaf719, entries=200, sequenceid=724, filesize=14.4 K 2024-12-06T14:43:08,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/b6f9153d57274479980299cdcd13b756 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b6f9153d57274479980299cdcd13b756 2024-12-06T14:43:08,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b6f9153d57274479980299cdcd13b756, entries=150, sequenceid=724, filesize=12.0 K 2024-12-06T14:43:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/770885aed34e4d168f6179721ed9fae7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/770885aed34e4d168f6179721ed9fae7 2024-12-06T14:43:08,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/770885aed34e4d168f6179721ed9fae7, entries=150, sequenceid=724, filesize=12.0 K 2024-12-06T14:43:08,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4828895d58f7749c326fda40500dce6a in 1281ms, sequenceid=724, compaction requested=true 2024-12-06T14:43:08,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:08,452 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:08,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:08,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:08,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:08,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:08,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:08,453 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:08,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:08,453 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:08,453 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:08,453 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/A is initiating minor compaction (all files) 2024-12-06T14:43:08,453 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/B is initiating minor compaction (all files) 2024-12-06T14:43:08,453 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/A in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,454 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/756deb9f7d2047b4a39ddf624a5107f9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/8e2787cdd278413e9d3880389be17272, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/708fe60b91d64e81bf5d15cb18eaf719] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=39.3 K 2024-12-06T14:43:08,454 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/B in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,454 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bc9db2bfc83443779529edc14db33c56, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3ebd2ea7e6f54aa38e43e813de4e7127, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b6f9153d57274479980299cdcd13b756] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.9 K 2024-12-06T14:43:08,455 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting bc9db2bfc83443779529edc14db33c56, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733496183833 2024-12-06T14:43:08,455 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756deb9f7d2047b4a39ddf624a5107f9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733496183833 2024-12-06T14:43:08,455 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebd2ea7e6f54aa38e43e813de4e7127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=706, earliestPutTs=1733496184406 2024-12-06T14:43:08,455 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e2787cdd278413e9d3880389be17272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=706, earliestPutTs=1733496184406 2024-12-06T14:43:08,456 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f9153d57274479980299cdcd13b756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1733496186016 2024-12-06T14:43:08,456 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708fe60b91d64e81bf5d15cb18eaf719, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1733496186016 2024-12-06T14:43:08,474 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:08,475 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/be6948ff56394dafab8d25b9f75e4f36 is 50, key is test_row_0/B:col10/1733496186016/Put/seqid=0 2024-12-06T14:43:08,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,481 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:08,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:08,486 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:08,487 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4f165c91e6e24af6a47a51c79be769d8 is 50, key is test_row_0/A:col10/1733496186016/Put/seqid=0 2024-12-06T14:43:08,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/e0580710285d40f0a3c8f01c2c4b6aa0 is 50, key is test_row_0/A:col10/1733496187213/Put/seqid=0 2024-12-06T14:43:08,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742116_1292 (size=13255) 2024-12-06T14:43:08,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742117_1293 (size=13255) 2024-12-06T14:43:08,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742118_1294 (size=12301) 2024-12-06T14:43:08,544 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/e0580710285d40f0a3c8f01c2c4b6aa0 2024-12-06T14:43:08,547 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T14:43:08,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/a461dc6d8dab4f34850a4fbfcad16dd6 is 50, key is test_row_0/B:col10/1733496187213/Put/seqid=0 2024-12-06T14:43:08,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742119_1295 (size=12301) 2024-12-06T14:43:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. as already flushing 2024-12-06T14:43:08,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496248599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496248602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496248602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496248708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496248708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496248708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,915 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/be6948ff56394dafab8d25b9f75e4f36 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/be6948ff56394dafab8d25b9f75e4f36 2024-12-06T14:43:08,919 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/B of 4828895d58f7749c326fda40500dce6a into be6948ff56394dafab8d25b9f75e4f36(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:08,919 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:08,919 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/B, priority=13, startTime=1733496188453; duration=0sec 2024-12-06T14:43:08,919 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:08,919 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:43:08,919 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:08,920 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:08,920 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 4828895d58f7749c326fda40500dce6a/C is initiating minor compaction (all files) 2024-12-06T14:43:08,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,920 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4828895d58f7749c326fda40500dce6a/C in TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,921 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/9a286bb5b488439499daddb0c6adb4b2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/60e646e5804f4b24b98abefa50b03383, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/770885aed34e4d168f6179721ed9fae7] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp, totalSize=36.9 K 2024-12-06T14:43:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1733496248918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,921 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a286bb5b488439499daddb0c6adb4b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733496183833 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1733496248918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:08,921 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e646e5804f4b24b98abefa50b03383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=706, earliestPutTs=1733496184406 2024-12-06T14:43:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38658 deadline: 1733496248918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:08,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 770885aed34e4d168f6179721ed9fae7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1733496186016 2024-12-06T14:43:08,929 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4828895d58f7749c326fda40500dce6a#C#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:08,929 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/2653666752a943d0bf152f7fb8d3622a is 50, key is test_row_0/C:col10/1733496186016/Put/seqid=0 2024-12-06T14:43:08,931 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/4f165c91e6e24af6a47a51c79be769d8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4f165c91e6e24af6a47a51c79be769d8 2024-12-06T14:43:08,936 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/A of 4828895d58f7749c326fda40500dce6a into 4f165c91e6e24af6a47a51c79be769d8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:08,936 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:08,936 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/A, priority=13, startTime=1733496188452; duration=0sec 2024-12-06T14:43:08,936 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:08,936 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:43:08,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742120_1296 (size=13255) 2024-12-06T14:43:08,966 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/a461dc6d8dab4f34850a4fbfcad16dd6 2024-12-06T14:43:08,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/c45d90ead3e4402293d27af2a64a396d is 50, key is test_row_0/C:col10/1733496187213/Put/seqid=0 2024-12-06T14:43:08,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742121_1297 (size=12301) 2024-12-06T14:43:08,978 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/c45d90ead3e4402293d27af2a64a396d 2024-12-06T14:43:08,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/e0580710285d40f0a3c8f01c2c4b6aa0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/e0580710285d40f0a3c8f01c2c4b6aa0 2024-12-06T14:43:08,986 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/e0580710285d40f0a3c8f01c2c4b6aa0, entries=150, sequenceid=741, filesize=12.0 K 2024-12-06T14:43:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/a461dc6d8dab4f34850a4fbfcad16dd6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a461dc6d8dab4f34850a4fbfcad16dd6 2024-12-06T14:43:08,990 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a461dc6d8dab4f34850a4fbfcad16dd6, entries=150, sequenceid=741, filesize=12.0 K 2024-12-06T14:43:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/c45d90ead3e4402293d27af2a64a396d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c45d90ead3e4402293d27af2a64a396d 2024-12-06T14:43:08,995 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c45d90ead3e4402293d27af2a64a396d, entries=150, sequenceid=741, filesize=12.0 K 2024-12-06T14:43:08,996 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4828895d58f7749c326fda40500dce6a in 515ms, sequenceid=741, compaction requested=false 2024-12-06T14:43:08,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:08,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:08,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-06T14:43:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-06T14:43:08,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-06T14:43:08,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7410 sec 2024-12-06T14:43:09,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7460 sec 2024-12-06T14:43:09,192 INFO [Thread-1005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1005. 2024-12-06T14:43:09,192 DEBUG [Thread-1005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,192 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,192 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,193 INFO [Thread-997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-997. 2024-12-06T14:43:09,193 DEBUG [Thread-997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,193 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,193 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,193 INFO [Thread-999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-999. 2024-12-06T14:43:09,194 DEBUG [Thread-999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,194 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,194 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,194 INFO [Thread-1001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1001. 2024-12-06T14:43:09,194 DEBUG [Thread-1001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,194 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,195 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,195 INFO [Thread-1003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1003. 2024-12-06T14:43:09,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,195 DEBUG [Thread-1003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,195 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,195 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:09,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:43:09,231 INFO [Thread-988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-988. 2024-12-06T14:43:09,231 DEBUG [Thread-988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,231 DEBUG [Thread-988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,231 INFO [Thread-994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-994. 2024-12-06T14:43:09,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:09,231 DEBUG [Thread-994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,231 INFO [Thread-986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-986. 2024-12-06T14:43:09,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:09,231 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,231 DEBUG [Thread-986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,231 DEBUG [Thread-988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:09,231 DEBUG [Thread-986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:09,231 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:09,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,232 DEBUG [Thread-986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:09,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/9aedd119467343c2bb2c0fa9adf70ea1 is 50, key is test_row_0/A:col10/1733496189229/Put/seqid=0 2024-12-06T14:43:09,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742122_1298 (size=12301) 2024-12-06T14:43:09,349 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/2653666752a943d0bf152f7fb8d3622a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2653666752a943d0bf152f7fb8d3622a 2024-12-06T14:43:09,354 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4828895d58f7749c326fda40500dce6a/C of 4828895d58f7749c326fda40500dce6a into 2653666752a943d0bf152f7fb8d3622a(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:09,354 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:09,354 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., storeName=4828895d58f7749c326fda40500dce6a/C, priority=13, startTime=1733496188453; duration=0sec 2024-12-06T14:43:09,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:09,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:43:09,370 INFO [Thread-990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-990. 2024-12-06T14:43:09,370 INFO [Thread-992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-992. 2024-12-06T14:43:09,370 DEBUG [Thread-992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,370 DEBUG [Thread-990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:09,370 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,370 DEBUG [Thread-990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,370 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,370 DEBUG [Thread-990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T14:43:09,399 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 181 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3322 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3272 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3256 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3338 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3292 2024-12-06T14:43:09,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T14:43:09,399 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:43:09,399 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:09,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:09,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:09,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T14:43:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T14:43:09,406 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496189406"}]},"ts":"1733496189406"} 2024-12-06T14:43:09,409 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T14:43:09,409 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T14:43:09,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:43:09,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, UNASSIGN}] 2024-12-06T14:43:09,413 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, UNASSIGN 2024-12-06T14:43:09,415 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=4828895d58f7749c326fda40500dce6a, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:09,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, UNASSIGN because future has completed 2024-12-06T14:43:09,417 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:43:09,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:43:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T14:43:09,570 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:09,570 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:43:09,570 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 4828895d58f7749c326fda40500dce6a, disabling compactions & flushes 2024-12-06T14:43:09,571 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:09,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/9aedd119467343c2bb2c0fa9adf70ea1 2024-12-06T14:43:09,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/4da73624829e45b5bc983b03b9e9d009 is 50, key is test_row_0/B:col10/1733496189229/Put/seqid=0 2024-12-06T14:43:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742123_1299 (size=12301) 2024-12-06T14:43:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T14:43:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T14:43:10,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/4da73624829e45b5bc983b03b9e9d009 2024-12-06T14:43:10,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/066d7f37a8e741c9b31b92f8363f1048 is 50, key is test_row_0/C:col10/1733496189229/Put/seqid=0 2024-12-06T14:43:10,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742124_1300 (size=12301) 2024-12-06T14:43:10,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/066d7f37a8e741c9b31b92f8363f1048 2024-12-06T14:43:10,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/9aedd119467343c2bb2c0fa9adf70ea1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9aedd119467343c2bb2c0fa9adf70ea1 2024-12-06T14:43:10,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9aedd119467343c2bb2c0fa9adf70ea1, entries=150, sequenceid=761, filesize=12.0 K 2024-12-06T14:43:10,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/4da73624829e45b5bc983b03b9e9d009 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/4da73624829e45b5bc983b03b9e9d009 2024-12-06T14:43:10,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/4da73624829e45b5bc983b03b9e9d009, entries=150, sequenceid=761, filesize=12.0 K 2024-12-06T14:43:10,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/066d7f37a8e741c9b31b92f8363f1048 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/066d7f37a8e741c9b31b92f8363f1048 2024-12-06T14:43:10,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/066d7f37a8e741c9b31b92f8363f1048, entries=150, sequenceid=761, filesize=12.0 K 2024-12-06T14:43:10,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for 4828895d58f7749c326fda40500dce6a in 1258ms, sequenceid=761, compaction requested=true 2024-12-06T14:43:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4828895d58f7749c326fda40500dce6a: 2024-12-06T14:43:10,490 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. after waiting 0 ms 2024-12-06T14:43:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4828895d58f7749c326fda40500dce6a:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:10,490 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 4828895d58f7749c326fda40500dce6a 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-06T14:43:10,490 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. because compaction request was cancelled 2024-12-06T14:43:10,490 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:A 2024-12-06T14:43:10,490 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. because compaction request was cancelled 2024-12-06T14:43:10,490 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:C 2024-12-06T14:43:10,490 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. because compaction request was cancelled 2024-12-06T14:43:10,490 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4828895d58f7749c326fda40500dce6a:B 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=A 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=B 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4828895d58f7749c326fda40500dce6a, store=C 2024-12-06T14:43:10,490 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:10,493 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/864435e43d11476e9ae37ffe4e0e5268 is 50, key is test_row_1/A:col10/1733496189369/Put/seqid=0 2024-12-06T14:43:10,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742125_1301 (size=9857) 2024-12-06T14:43:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T14:43:10,898 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/864435e43d11476e9ae37ffe4e0e5268 2024-12-06T14:43:10,905 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/f91e15f7a60c4934aadb4dc2cd09d2a5 is 50, key is test_row_1/B:col10/1733496189369/Put/seqid=0 2024-12-06T14:43:10,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742126_1302 (size=9857) 2024-12-06T14:43:11,309 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/f91e15f7a60c4934aadb4dc2cd09d2a5 2024-12-06T14:43:11,318 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/76b56e4d6bbc44468f3a20f6d8e43c3c is 50, key is test_row_1/C:col10/1733496189369/Put/seqid=0 2024-12-06T14:43:11,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742127_1303 (size=9857) 2024-12-06T14:43:11,324 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/76b56e4d6bbc44468f3a20f6d8e43c3c 2024-12-06T14:43:11,332 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/A/864435e43d11476e9ae37ffe4e0e5268 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/864435e43d11476e9ae37ffe4e0e5268 2024-12-06T14:43:11,339 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/864435e43d11476e9ae37ffe4e0e5268, entries=100, sequenceid=767, filesize=9.6 K 2024-12-06T14:43:11,341 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/B/f91e15f7a60c4934aadb4dc2cd09d2a5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f91e15f7a60c4934aadb4dc2cd09d2a5 2024-12-06T14:43:11,347 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f91e15f7a60c4934aadb4dc2cd09d2a5, entries=100, sequenceid=767, filesize=9.6 K 2024-12-06T14:43:11,348 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/.tmp/C/76b56e4d6bbc44468f3a20f6d8e43c3c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/76b56e4d6bbc44468f3a20f6d8e43c3c 2024-12-06T14:43:11,353 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/76b56e4d6bbc44468f3a20f6d8e43c3c, entries=100, sequenceid=767, filesize=9.6 K 2024-12-06T14:43:11,354 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 4828895d58f7749c326fda40500dce6a in 864ms, sequenceid=767, compaction requested=true 2024-12-06T14:43:11,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/db59a63c08be4a139ec5190b6aa68a57, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/df263c7f66154d28bb53e22c61095f1e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/fcbc8ddd283b4b4e8241e1e233aadcc2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/98678d6622f04550a56b4643867539f1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/d4fc467b66974979a27fd8a563aa182c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/53862087b7194a6280ab50f1eab197c8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/a2f826e56944477c9d562497efaa7ae5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/783e306ee19f4db7b2c48aa5362b77b8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/bb2da9b6b48a48158ee7d4a6e7bd872e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/6a0fd27d318a463fb23d24b982646645, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/550317f6a3334bdb82626942a5748cd3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4abad3cc392642c197f01c267a5666a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/92ec45283468459a8e70dadd97b21dca, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ed842bb1fd2e465580afd0db371d208f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28e6d7b8bc2a46498ad3c4caad036372, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/7d568fd7b54b474dac4393c83229fd83, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4d6f41e677454afb89497b7278591ab3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ad3317d2aee1482cab256dbf05d9cf17, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b40abb48e14948959bc7abf9766bacfc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28a8a83f846942c1887872d15ce72a71, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b702c8d4383947458749da927deba369, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9406f5902d124c5b8355b6ab3d0db1fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ca6d658189534e0a831bc4d5b0698a38, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/756deb9f7d2047b4a39ddf624a5107f9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/85ef1f5e334949b1ab3d096ce306e6d8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/8e2787cdd278413e9d3880389be17272, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/708fe60b91d64e81bf5d15cb18eaf719] to archive 2024-12-06T14:43:11,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:43:11,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/db59a63c08be4a139ec5190b6aa68a57 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/db59a63c08be4a139ec5190b6aa68a57 2024-12-06T14:43:11,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/df263c7f66154d28bb53e22c61095f1e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/df263c7f66154d28bb53e22c61095f1e 2024-12-06T14:43:11,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/fcbc8ddd283b4b4e8241e1e233aadcc2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/fcbc8ddd283b4b4e8241e1e233aadcc2 2024-12-06T14:43:11,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/98678d6622f04550a56b4643867539f1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/98678d6622f04550a56b4643867539f1 2024-12-06T14:43:11,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/d4fc467b66974979a27fd8a563aa182c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/d4fc467b66974979a27fd8a563aa182c 2024-12-06T14:43:11,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/53862087b7194a6280ab50f1eab197c8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/53862087b7194a6280ab50f1eab197c8 2024-12-06T14:43:11,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/a2f826e56944477c9d562497efaa7ae5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/a2f826e56944477c9d562497efaa7ae5 2024-12-06T14:43:11,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/783e306ee19f4db7b2c48aa5362b77b8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/783e306ee19f4db7b2c48aa5362b77b8 2024-12-06T14:43:11,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/bb2da9b6b48a48158ee7d4a6e7bd872e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/bb2da9b6b48a48158ee7d4a6e7bd872e 2024-12-06T14:43:11,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/6a0fd27d318a463fb23d24b982646645 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/6a0fd27d318a463fb23d24b982646645 2024-12-06T14:43:11,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/550317f6a3334bdb82626942a5748cd3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/550317f6a3334bdb82626942a5748cd3 2024-12-06T14:43:11,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4abad3cc392642c197f01c267a5666a5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4abad3cc392642c197f01c267a5666a5 2024-12-06T14:43:11,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/92ec45283468459a8e70dadd97b21dca to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/92ec45283468459a8e70dadd97b21dca 2024-12-06T14:43:11,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/3e45e5d1c49a4fb5b09f8a8a7e19b0bd 2024-12-06T14:43:11,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ed842bb1fd2e465580afd0db371d208f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ed842bb1fd2e465580afd0db371d208f 2024-12-06T14:43:11,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28e6d7b8bc2a46498ad3c4caad036372 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28e6d7b8bc2a46498ad3c4caad036372 2024-12-06T14:43:11,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/7d568fd7b54b474dac4393c83229fd83 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/7d568fd7b54b474dac4393c83229fd83 2024-12-06T14:43:11,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4d6f41e677454afb89497b7278591ab3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4d6f41e677454afb89497b7278591ab3 2024-12-06T14:43:11,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ad3317d2aee1482cab256dbf05d9cf17 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ad3317d2aee1482cab256dbf05d9cf17 2024-12-06T14:43:11,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b40abb48e14948959bc7abf9766bacfc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b40abb48e14948959bc7abf9766bacfc 2024-12-06T14:43:11,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28a8a83f846942c1887872d15ce72a71 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/28a8a83f846942c1887872d15ce72a71 2024-12-06T14:43:11,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b702c8d4383947458749da927deba369 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/b702c8d4383947458749da927deba369 2024-12-06T14:43:11,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9406f5902d124c5b8355b6ab3d0db1fa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9406f5902d124c5b8355b6ab3d0db1fa 2024-12-06T14:43:11,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ca6d658189534e0a831bc4d5b0698a38 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/ca6d658189534e0a831bc4d5b0698a38 2024-12-06T14:43:11,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/756deb9f7d2047b4a39ddf624a5107f9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/756deb9f7d2047b4a39ddf624a5107f9 2024-12-06T14:43:11,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/85ef1f5e334949b1ab3d096ce306e6d8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/85ef1f5e334949b1ab3d096ce306e6d8 2024-12-06T14:43:11,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/8e2787cdd278413e9d3880389be17272 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/8e2787cdd278413e9d3880389be17272 2024-12-06T14:43:11,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/708fe60b91d64e81bf5d15cb18eaf719 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/708fe60b91d64e81bf5d15cb18eaf719 2024-12-06T14:43:11,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/d30052fd3dab46b79bd96a2b84e7e383, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14faec9c51ca4d868f38228eefd19e0b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/84dbda18f81b415385a1e412c99f6657, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/36b54a855404493abc81945aa922027a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b639d9028f424fc584e770dd2ca198c6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/83714f8c28bf476ba8ed36c136923063, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e0e28816dbe24024af870217f9729042, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/2fa0685c6b884efd8ca2b033f07160a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a07c23f772ba4438bde93d702acc1f9a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f5e159772d71419eb372cd796d22a501, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3cbe53d0d4cf49aba07746122cba0851, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3bc127dab8f44e869e6f3e76719c5a3d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/7a8de30c7d614117a278c30b0c62f788, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bb1f248efa334559b7283143c9c555b1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/63866df553b841f49961671e8861b4a9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e58045cf69cc49008e6b57e68f3a483e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1024fb763aa24af4aedb6e4a71158224, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/aa3aa6e7ce90442498fefe05fad003fe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/dffa18f6c144475b9d590507e39925ac, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1f495db7bdd64477b173f81e7dbae164, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/eadef417db47422a900df0ce6ff7604d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/cd3e3f2555024bf891b7986ac9613440, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14defd1a3a0641bf942cd99ebaff48c0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e2f8f0c8d8184bae8750346f4967ded6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bc9db2bfc83443779529edc14db33c56, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/8d9d968e989747cba66b713f9dc21a59, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3ebd2ea7e6f54aa38e43e813de4e7127, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b6f9153d57274479980299cdcd13b756] to archive 2024-12-06T14:43:11,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:43:11,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/d30052fd3dab46b79bd96a2b84e7e383 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/d30052fd3dab46b79bd96a2b84e7e383 2024-12-06T14:43:11,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14faec9c51ca4d868f38228eefd19e0b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14faec9c51ca4d868f38228eefd19e0b 2024-12-06T14:43:11,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/84dbda18f81b415385a1e412c99f6657 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/84dbda18f81b415385a1e412c99f6657 2024-12-06T14:43:11,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/36b54a855404493abc81945aa922027a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/36b54a855404493abc81945aa922027a 2024-12-06T14:43:11,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b639d9028f424fc584e770dd2ca198c6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b639d9028f424fc584e770dd2ca198c6 2024-12-06T14:43:11,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/83714f8c28bf476ba8ed36c136923063 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/83714f8c28bf476ba8ed36c136923063 2024-12-06T14:43:11,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e0e28816dbe24024af870217f9729042 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e0e28816dbe24024af870217f9729042 2024-12-06T14:43:11,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/2fa0685c6b884efd8ca2b033f07160a5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/2fa0685c6b884efd8ca2b033f07160a5 2024-12-06T14:43:11,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a07c23f772ba4438bde93d702acc1f9a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a07c23f772ba4438bde93d702acc1f9a 2024-12-06T14:43:11,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f5e159772d71419eb372cd796d22a501 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f5e159772d71419eb372cd796d22a501 2024-12-06T14:43:11,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3cbe53d0d4cf49aba07746122cba0851 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3cbe53d0d4cf49aba07746122cba0851 2024-12-06T14:43:11,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3bc127dab8f44e869e6f3e76719c5a3d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3bc127dab8f44e869e6f3e76719c5a3d 2024-12-06T14:43:11,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/7a8de30c7d614117a278c30b0c62f788 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/7a8de30c7d614117a278c30b0c62f788 2024-12-06T14:43:11,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bb1f248efa334559b7283143c9c555b1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bb1f248efa334559b7283143c9c555b1 2024-12-06T14:43:11,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/63866df553b841f49961671e8861b4a9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/63866df553b841f49961671e8861b4a9 2024-12-06T14:43:11,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e58045cf69cc49008e6b57e68f3a483e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e58045cf69cc49008e6b57e68f3a483e 2024-12-06T14:43:11,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1024fb763aa24af4aedb6e4a71158224 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1024fb763aa24af4aedb6e4a71158224 2024-12-06T14:43:11,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/aa3aa6e7ce90442498fefe05fad003fe to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/aa3aa6e7ce90442498fefe05fad003fe 2024-12-06T14:43:11,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/dffa18f6c144475b9d590507e39925ac to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/dffa18f6c144475b9d590507e39925ac 2024-12-06T14:43:11,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1f495db7bdd64477b173f81e7dbae164 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/1f495db7bdd64477b173f81e7dbae164 2024-12-06T14:43:11,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/eadef417db47422a900df0ce6ff7604d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/eadef417db47422a900df0ce6ff7604d 2024-12-06T14:43:11,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/cd3e3f2555024bf891b7986ac9613440 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/cd3e3f2555024bf891b7986ac9613440 2024-12-06T14:43:11,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14defd1a3a0641bf942cd99ebaff48c0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/14defd1a3a0641bf942cd99ebaff48c0 2024-12-06T14:43:11,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e2f8f0c8d8184bae8750346f4967ded6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/e2f8f0c8d8184bae8750346f4967ded6 2024-12-06T14:43:11,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bc9db2bfc83443779529edc14db33c56 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/bc9db2bfc83443779529edc14db33c56 2024-12-06T14:43:11,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/8d9d968e989747cba66b713f9dc21a59 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/8d9d968e989747cba66b713f9dc21a59 2024-12-06T14:43:11,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3ebd2ea7e6f54aa38e43e813de4e7127 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/3ebd2ea7e6f54aa38e43e813de4e7127 2024-12-06T14:43:11,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b6f9153d57274479980299cdcd13b756 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/b6f9153d57274479980299cdcd13b756 2024-12-06T14:43:11,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e0a7731da1e8423e87f0212a5e0e159d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a1efafc8cd194af1bfae87b76ec045d8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/555f98f6ffbd4c37a30413fff46d2a1a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/93665f6451c14c2a8eca599e71698f5e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1fba0aa077f14169b178beb5cca8fa60, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1215914dcc6945ce9b12abda07220c23, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/34e67ae21a04407db33e3d628b4cfef2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2980861e962040f7af20a2e8d0e7b921, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/3357006aa19847d2a44a068b17c74258, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ff6fb81a9d884a27a03a2f71b541c899, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/61ebaa4023924fe8abcf34489f83dc29, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c8f82a7bc34949e9b42a49eb45ef5fe7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a147ac6e3d2c47f2a085d3f5f05a7f42, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e6a5740652f9476e899d0aafdefae0a5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/05b2644b72d74d8cab83af6e87933ec2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/195164aa526f4730be8cefaeb30f0150, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1d836000a9c04c118880e37ca7d36798, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e9a9fc678d574042b680c22cd6393087, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/879045276cb24f99a58c7196ee58072b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/dbfe166ec9814393a2f5c87cc167fe05, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a911f2bc2c3744a7b7a4e88e04b20172, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ee6dbf7213064bc699264928d96b8bf3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/0c80442cbf7249f9a9dc283f78590943, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/07eafe66f0394d0a9d109fd4e5483a5b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/9a286bb5b488439499daddb0c6adb4b2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/435210e422dd4ec6a6213aaa1b73f787, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/60e646e5804f4b24b98abefa50b03383, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/770885aed34e4d168f6179721ed9fae7] to archive 2024-12-06T14:43:11,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:43:11,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e0a7731da1e8423e87f0212a5e0e159d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e0a7731da1e8423e87f0212a5e0e159d 2024-12-06T14:43:11,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a1efafc8cd194af1bfae87b76ec045d8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a1efafc8cd194af1bfae87b76ec045d8 2024-12-06T14:43:11,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/555f98f6ffbd4c37a30413fff46d2a1a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/555f98f6ffbd4c37a30413fff46d2a1a 2024-12-06T14:43:11,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/93665f6451c14c2a8eca599e71698f5e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/93665f6451c14c2a8eca599e71698f5e 2024-12-06T14:43:11,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1fba0aa077f14169b178beb5cca8fa60 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1fba0aa077f14169b178beb5cca8fa60 2024-12-06T14:43:11,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1215914dcc6945ce9b12abda07220c23 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1215914dcc6945ce9b12abda07220c23 2024-12-06T14:43:11,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/34e67ae21a04407db33e3d628b4cfef2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/34e67ae21a04407db33e3d628b4cfef2 2024-12-06T14:43:11,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2980861e962040f7af20a2e8d0e7b921 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2980861e962040f7af20a2e8d0e7b921 2024-12-06T14:43:11,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/3357006aa19847d2a44a068b17c74258 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/3357006aa19847d2a44a068b17c74258 2024-12-06T14:43:11,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ff6fb81a9d884a27a03a2f71b541c899 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ff6fb81a9d884a27a03a2f71b541c899 2024-12-06T14:43:11,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/61ebaa4023924fe8abcf34489f83dc29 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/61ebaa4023924fe8abcf34489f83dc29 2024-12-06T14:43:11,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c8f82a7bc34949e9b42a49eb45ef5fe7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c8f82a7bc34949e9b42a49eb45ef5fe7 2024-12-06T14:43:11,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a147ac6e3d2c47f2a085d3f5f05a7f42 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a147ac6e3d2c47f2a085d3f5f05a7f42 2024-12-06T14:43:11,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e6a5740652f9476e899d0aafdefae0a5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e6a5740652f9476e899d0aafdefae0a5 2024-12-06T14:43:11,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/05b2644b72d74d8cab83af6e87933ec2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/05b2644b72d74d8cab83af6e87933ec2 2024-12-06T14:43:11,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/195164aa526f4730be8cefaeb30f0150 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/195164aa526f4730be8cefaeb30f0150 2024-12-06T14:43:11,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1d836000a9c04c118880e37ca7d36798 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/1d836000a9c04c118880e37ca7d36798 2024-12-06T14:43:11,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e9a9fc678d574042b680c22cd6393087 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/e9a9fc678d574042b680c22cd6393087 2024-12-06T14:43:11,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/879045276cb24f99a58c7196ee58072b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/879045276cb24f99a58c7196ee58072b 2024-12-06T14:43:11,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/dbfe166ec9814393a2f5c87cc167fe05 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/dbfe166ec9814393a2f5c87cc167fe05 2024-12-06T14:43:11,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a911f2bc2c3744a7b7a4e88e04b20172 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/a911f2bc2c3744a7b7a4e88e04b20172 2024-12-06T14:43:11,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ee6dbf7213064bc699264928d96b8bf3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/ee6dbf7213064bc699264928d96b8bf3 2024-12-06T14:43:11,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/0c80442cbf7249f9a9dc283f78590943 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/0c80442cbf7249f9a9dc283f78590943 2024-12-06T14:43:11,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/07eafe66f0394d0a9d109fd4e5483a5b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/07eafe66f0394d0a9d109fd4e5483a5b 2024-12-06T14:43:11,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/9a286bb5b488439499daddb0c6adb4b2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/9a286bb5b488439499daddb0c6adb4b2 2024-12-06T14:43:11,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/435210e422dd4ec6a6213aaa1b73f787 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/435210e422dd4ec6a6213aaa1b73f787 2024-12-06T14:43:11,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/60e646e5804f4b24b98abefa50b03383 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/60e646e5804f4b24b98abefa50b03383 2024-12-06T14:43:11,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/770885aed34e4d168f6179721ed9fae7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/770885aed34e4d168f6179721ed9fae7 2024-12-06T14:43:11,507 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/recovered.edits/770.seqid, newMaxSeqId=770, maxSeqId=1 2024-12-06T14:43:11,508 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. 2024-12-06T14:43:11,508 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 4828895d58f7749c326fda40500dce6a: Waiting for close lock at 1733496189570Running coprocessor pre-close hooks at 1733496189570Disabling compacts and flushes for region at 1733496189570Disabling writes for close at 1733496190490 (+920 ms)Obtaining lock to block concurrent updates at 1733496190490Preparing flush snapshotting stores in 4828895d58f7749c326fda40500dce6a at 1733496190490Finished memstore snapshotting TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733496190490Flushing stores of TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a. at 1733496190491 (+1 ms)Flushing 4828895d58f7749c326fda40500dce6a/A: creating writer at 1733496190491Flushing 4828895d58f7749c326fda40500dce6a/A: appending metadata at 1733496190493 (+2 ms)Flushing 4828895d58f7749c326fda40500dce6a/A: closing flushed file at 1733496190493Flushing 4828895d58f7749c326fda40500dce6a/B: creating writer at 1733496190902 (+409 ms)Flushing 4828895d58f7749c326fda40500dce6a/B: appending metadata at 1733496190905 (+3 ms)Flushing 4828895d58f7749c326fda40500dce6a/B: closing flushed file at 1733496190905Flushing 4828895d58f7749c326fda40500dce6a/C: creating writer at 1733496191314 (+409 ms)Flushing 4828895d58f7749c326fda40500dce6a/C: appending metadata at 1733496191318 (+4 ms)Flushing 4828895d58f7749c326fda40500dce6a/C: closing flushed file at 1733496191318Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26622b9a: reopening flushed file at 1733496191331 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1da8a8fc: reopening flushed file at 1733496191339 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@725dfee0: reopening flushed file at 1733496191347 (+8 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 4828895d58f7749c326fda40500dce6a in 864ms, sequenceid=767, compaction requested=true at 1733496191354 (+7 ms)Writing region close event to WAL at 1733496191504 (+150 ms)Running coprocessor post-close hooks at 1733496191508 (+4 ms)Closed at 1733496191508 2024-12-06T14:43:11,510 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:11,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=4828895d58f7749c326fda40500dce6a, regionState=CLOSED 2024-12-06T14:43:11,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:43:11,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-06T14:43:11,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 4828895d58f7749c326fda40500dce6a, server=bb7ea3e46d9c,35831,1733496100253 in 2.0960 sec 2024-12-06T14:43:11,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-06T14:43:11,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4828895d58f7749c326fda40500dce6a, UNASSIGN in 2.1030 sec 2024-12-06T14:43:11,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-06T14:43:11,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1070 sec 2024-12-06T14:43:11,520 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496191519"}]},"ts":"1733496191519"} 2024-12-06T14:43:11,521 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T14:43:11,521 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T14:43:11,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1210 sec 2024-12-06T14:43:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T14:43:11,549 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:11,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T14:43:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,551 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T14:43:11,552 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,553 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:11,556 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/recovered.edits] 2024-12-06T14:43:11,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4f165c91e6e24af6a47a51c79be769d8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/4f165c91e6e24af6a47a51c79be769d8 2024-12-06T14:43:11,559 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/864435e43d11476e9ae37ffe4e0e5268 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/864435e43d11476e9ae37ffe4e0e5268 2024-12-06T14:43:11,560 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9aedd119467343c2bb2c0fa9adf70ea1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/9aedd119467343c2bb2c0fa9adf70ea1 2024-12-06T14:43:11,562 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/e0580710285d40f0a3c8f01c2c4b6aa0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/A/e0580710285d40f0a3c8f01c2c4b6aa0 2024-12-06T14:43:11,564 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/4da73624829e45b5bc983b03b9e9d009 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/4da73624829e45b5bc983b03b9e9d009 2024-12-06T14:43:11,565 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a461dc6d8dab4f34850a4fbfcad16dd6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/a461dc6d8dab4f34850a4fbfcad16dd6 2024-12-06T14:43:11,567 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/be6948ff56394dafab8d25b9f75e4f36 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/be6948ff56394dafab8d25b9f75e4f36 2024-12-06T14:43:11,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f91e15f7a60c4934aadb4dc2cd09d2a5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/B/f91e15f7a60c4934aadb4dc2cd09d2a5 2024-12-06T14:43:11,571 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/066d7f37a8e741c9b31b92f8363f1048 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/066d7f37a8e741c9b31b92f8363f1048 2024-12-06T14:43:11,572 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2653666752a943d0bf152f7fb8d3622a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/2653666752a943d0bf152f7fb8d3622a 2024-12-06T14:43:11,573 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/76b56e4d6bbc44468f3a20f6d8e43c3c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/76b56e4d6bbc44468f3a20f6d8e43c3c 2024-12-06T14:43:11,574 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c45d90ead3e4402293d27af2a64a396d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/C/c45d90ead3e4402293d27af2a64a396d 2024-12-06T14:43:11,578 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/recovered.edits/770.seqid to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a/recovered.edits/770.seqid 2024-12-06T14:43:11,579 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/4828895d58f7749c326fda40500dce6a 2024-12-06T14:43:11,579 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T14:43:11,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,584 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T14:43:11,586 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T14:43:11,588 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,588 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T14:43:11,588 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733496191588"}]},"ts":"9223372036854775807"} 2024-12-06T14:43:11,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T14:43:11,590 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4828895d58f7749c326fda40500dce6a, NAME => 'TestAcidGuarantees,,1733496167969.4828895d58f7749c326fda40500dce6a.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T14:43:11,590 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T14:43:11,590 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733496191590"}]},"ts":"9223372036854775807"} 2024-12-06T14:43:11,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T14:43:11,593 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-06T14:43:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T14:43:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T14:43:11,659 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:11,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_912585382_22 at /127.0.0.1:52260 [Waiting for operation #382] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1216782701_22 at /127.0.0.1:42036 [Waiting for operation #213] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=585 (was 488) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5597 (was 5776) 2024-12-06T14:43:11,680 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=585, ProcessCount=11, AvailableMemoryMB=5596 2024-12-06T14:43:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:43:11,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:11,684 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:43:11,684 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:11,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-06T14:43:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-06T14:43:11,685 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:43:11,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742128_1304 (size=1061) 2024-12-06T14:43:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-06T14:43:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-06T14:43:12,093 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:43:12,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742129_1305 (size=53) 2024-12-06T14:43:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-06T14:43:12,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:12,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 415d8e5bcca7a5e3eb923a4249401464, disabling compactions & flushes 2024-12-06T14:43:12,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. after waiting 0 ms 2024-12-06T14:43:12,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 415d8e5bcca7a5e3eb923a4249401464: Waiting for close lock at 1733496192499Disabling compacts and flushes for region at 1733496192499Disabling writes for close at 1733496192499Writing region close event to WAL at 1733496192499Closed at 1733496192499 2024-12-06T14:43:12,501 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:43:12,501 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733496192501"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496192501"}]},"ts":"1733496192501"} 2024-12-06T14:43:12,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:43:12,506 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:43:12,506 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496192506"}]},"ts":"1733496192506"} 2024-12-06T14:43:12,508 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T14:43:12,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, ASSIGN}] 2024-12-06T14:43:12,509 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, ASSIGN 2024-12-06T14:43:12,510 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=false 2024-12-06T14:43:12,661 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:12,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, ASSIGN because future has completed 2024-12-06T14:43:12,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:43:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-06T14:43:12,822 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,823 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:12,823 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,823 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:12,823 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,823 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,825 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,826 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:12,827 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415d8e5bcca7a5e3eb923a4249401464 columnFamilyName A 2024-12-06T14:43:12,827 DEBUG [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:12,827 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(327): Store=415d8e5bcca7a5e3eb923a4249401464/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:12,828 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,829 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:12,829 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415d8e5bcca7a5e3eb923a4249401464 columnFamilyName B 2024-12-06T14:43:12,829 DEBUG [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:12,830 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(327): Store=415d8e5bcca7a5e3eb923a4249401464/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:12,830 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,831 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:12,831 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415d8e5bcca7a5e3eb923a4249401464 columnFamilyName C 2024-12-06T14:43:12,832 DEBUG [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:12,832 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(327): Store=415d8e5bcca7a5e3eb923a4249401464/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:12,832 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,832 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,834 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,834 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,835 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,835 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,836 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:43:12,838 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,840 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:12,841 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 415d8e5bcca7a5e3eb923a4249401464; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61449854, jitterRate=-0.08432582020759583}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:43:12,841 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:12,842 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 415d8e5bcca7a5e3eb923a4249401464: Running coprocessor pre-open hook at 1733496192823Writing region info on filesystem at 1733496192823Initializing all the Stores at 1733496192824 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496192824Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496192825 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496192825Cleaning up temporary data from old regions at 1733496192835 (+10 ms)Running coprocessor post-open hooks at 1733496192841 (+6 ms)Region opened successfully at 1733496192842 (+1 ms) 2024-12-06T14:43:12,843 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., pid=85, masterSystemTime=1733496192818 2024-12-06T14:43:12,845 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,845 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:12,846 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:12,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:43:12,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-06T14:43:12,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 in 183 msec 2024-12-06T14:43:12,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-06T14:43:12,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, ASSIGN in 342 msec 2024-12-06T14:43:12,853 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:43:12,853 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496192853"}]},"ts":"1733496192853"} 2024-12-06T14:43:12,854 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T14:43:12,855 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:43:12,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-12-06T14:43:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-06T14:43:13,829 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:13,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29820261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:13,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:13,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:13,833 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2a80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:13,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d59790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:13,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:13,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:13,839 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:43:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:43:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c7e2451 2024-12-06T14:43:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:43:13,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:43:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:43:13,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:13,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742130_1306 (size=1097) 2024-12-06T14:43:14,257 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-06T14:43:14,257 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-06T14:43:14,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:43:14,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, REOPEN/MOVE}] 2024-12-06T14:43:14,262 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, REOPEN/MOVE 2024-12-06T14:43:14,263 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, REOPEN/MOVE because future has completed 2024-12-06T14:43:14,265 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:43:14,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:43:14,417 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,417 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:43:14,418 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 415d8e5bcca7a5e3eb923a4249401464, disabling compactions & flushes 2024-12-06T14:43:14,418 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,418 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,418 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. after waiting 0 ms 2024-12-06T14:43:14,418 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,421 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T14:43:14,422 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,422 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 415d8e5bcca7a5e3eb923a4249401464: Waiting for close lock at 1733496194417Running coprocessor pre-close hooks at 1733496194417Disabling compacts and flushes for region at 1733496194417Disabling writes for close at 1733496194418 (+1 ms)Writing region close event to WAL at 1733496194418Running coprocessor post-close hooks at 1733496194422 (+4 ms)Closed at 1733496194422 2024-12-06T14:43:14,422 WARN [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 415d8e5bcca7a5e3eb923a4249401464 to self. 2024-12-06T14:43:14,424 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,425 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=CLOSED 2024-12-06T14:43:14,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:43:14,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-06T14:43:14,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 in 162 msec 2024-12-06T14:43:14,429 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, REOPEN/MOVE; state=CLOSED, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=true 2024-12-06T14:43:14,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, REOPEN/MOVE because future has completed 2024-12-06T14:43:14,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:43:14,736 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:14,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:14,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,738 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,739 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:14,740 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415d8e5bcca7a5e3eb923a4249401464 columnFamilyName A 2024-12-06T14:43:14,741 DEBUG [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:14,741 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(327): Store=415d8e5bcca7a5e3eb923a4249401464/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:14,742 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,743 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:14,743 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415d8e5bcca7a5e3eb923a4249401464 columnFamilyName B 2024-12-06T14:43:14,743 DEBUG [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:14,744 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(327): Store=415d8e5bcca7a5e3eb923a4249401464/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:14,744 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,745 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:14,745 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415d8e5bcca7a5e3eb923a4249401464 columnFamilyName C 2024-12-06T14:43:14,745 DEBUG [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:14,745 INFO [StoreOpener-415d8e5bcca7a5e3eb923a4249401464-1 {}] regionserver.HStore(327): Store=415d8e5bcca7a5e3eb923a4249401464/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:14,746 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,746 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,746 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,749 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,750 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,750 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,750 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:43:14,753 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,754 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 415d8e5bcca7a5e3eb923a4249401464; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72725462, jitterRate=0.08369383215904236}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:43:14,754 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,754 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 415d8e5bcca7a5e3eb923a4249401464: Running coprocessor pre-open hook at 1733496194736Writing region info on filesystem at 1733496194736Initializing all the Stores at 1733496194737 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496194737Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496194738 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496194738Cleaning up temporary data from old regions at 1733496194750 (+12 ms)Running coprocessor post-open hooks at 1733496194754 (+4 ms)Region opened successfully at 1733496194754 2024-12-06T14:43:14,755 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., pid=90, masterSystemTime=1733496194732 2024-12-06T14:43:14,757 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,757 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:14,757 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=OPEN, openSeqNum=5, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:43:14,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-06T14:43:14,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 in 179 msec 2024-12-06T14:43:14,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-06T14:43:14,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, REOPEN/MOVE in 500 msec 2024-12-06T14:43:14,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-06T14:43:14,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-12-06T14:43:14,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 930 msec 2024-12-06T14:43:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-06T14:43:14,778 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:14,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c07883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,780 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acf579b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,783 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29bc7da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48721c16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,796 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc73fd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,808 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7393505a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,811 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5bf7da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,814 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dbae8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,816 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7332a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,819 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c5ad92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,820 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3556c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469d248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,830 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c40d5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa7507c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8c61ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d92d30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,837 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a87c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a1007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284fc0f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0248fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,847 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c16a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,849 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c52868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:14,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:14,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2f4379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:14,855 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79540ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:14,857 DEBUG [Thread-1410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,858 DEBUG [Thread-1410 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,858 DEBUG [Thread-1410 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,860 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,860 DEBUG [Thread-1412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,864 DEBUG [Thread-1418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,865 DEBUG [Thread-1414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,868 DEBUG [Thread-1416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,876 DEBUG [Thread-1427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,877 DEBUG [Thread-1427 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,878 DEBUG [Thread-1427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:14,888 DEBUG [Thread-1429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,889 DEBUG [Thread-1429 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,890 DEBUG [Thread-1429 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,890 DEBUG [Thread-1421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,891 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:14,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:14,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:14,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-06T14:43:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T14:43:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:43:14,901 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:14,902 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:14,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:14,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120619adee677b5c4898a3b98fc93c647e94_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496194883/Put/seqid=0 2024-12-06T14:43:14,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733496194894, max timestamp=1733496194936Num uniques 859; 2024-12-06T14:43:14,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T14:43:14,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733496194894, max timestamp=1733496194936Num uniques 859; 2024-12-06T14:43:14,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T14:43:14,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733496194894, max timestamp=1733496194936Num uniques 859; 2024-12-06T14:43:14,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T14:43:14,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742131_1307 (size=17034) 2024-12-06T14:43:14,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:14,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120619adee677b5c4898a3b98fc93c647e94_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120619adee677b5c4898a3b98fc93c647e94_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:14,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3a11efba1a3c4a4eb408df8f9c5b7004, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:14,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3a11efba1a3c4a4eb408df8f9c5b7004 is 175, key is test_row_0/A:col10/1733496194883/Put/seqid=0 2024-12-06T14:43:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496254967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496254968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496254969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:14,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742132_1308 (size=48139) 2024-12-06T14:43:14,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3a11efba1a3c4a4eb408df8f9c5b7004 2024-12-06T14:43:14,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496254976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496254978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T14:43:15,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/2cf9d31c3d934e9cac724e1fb9e5fed4 is 50, key is test_row_0/B:col10/1733496194883/Put/seqid=0 2024-12-06T14:43:15,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742133_1309 (size=12001) 2024-12-06T14:43:15,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/2cf9d31c3d934e9cac724e1fb9e5fed4 2024-12-06T14:43:15,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T14:43:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/ea2e46c1750547e385135b47b432a618 is 50, key is test_row_0/C:col10/1733496194883/Put/seqid=0 2024-12-06T14:43:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742134_1310 (size=12001) 2024-12-06T14:43:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496255089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496255089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496255089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496255098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496255098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T14:43:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T14:43:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496255308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496255309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496255309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496255309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496255309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T14:43:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:15,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/ea2e46c1750547e385135b47b432a618 2024-12-06T14:43:15,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3a11efba1a3c4a4eb408df8f9c5b7004 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004 2024-12-06T14:43:15,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004, entries=250, sequenceid=15, filesize=47.0 K 2024-12-06T14:43:15,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/2cf9d31c3d934e9cac724e1fb9e5fed4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/2cf9d31c3d934e9cac724e1fb9e5fed4 2024-12-06T14:43:15,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/2cf9d31c3d934e9cac724e1fb9e5fed4, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T14:43:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/ea2e46c1750547e385135b47b432a618 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/ea2e46c1750547e385135b47b432a618 2024-12-06T14:43:15,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/ea2e46c1750547e385135b47b432a618, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T14:43:15,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 415d8e5bcca7a5e3eb923a4249401464 in 626ms, sequenceid=15, compaction requested=false 2024-12-06T14:43:15,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:15,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T14:43:15,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,518 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-06T14:43:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:15,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206915685828fe742b1855b68297ce61b1d_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496194976/Put/seqid=0 2024-12-06T14:43:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T14:43:15,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742135_1311 (size=12154) 2024-12-06T14:43:15,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,547 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206915685828fe742b1855b68297ce61b1d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206915685828fe742b1855b68297ce61b1d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ba2346b3df994a4eae5ae6139aa2d953, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ba2346b3df994a4eae5ae6139aa2d953 is 175, key is test_row_0/A:col10/1733496194976/Put/seqid=0 2024-12-06T14:43:15,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742136_1312 (size=30955) 2024-12-06T14:43:15,559 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ba2346b3df994a4eae5ae6139aa2d953 2024-12-06T14:43:15,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/f560eb546ae5419580509e478791bfa3 is 50, key is test_row_0/B:col10/1733496194976/Put/seqid=0 2024-12-06T14:43:15,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742137_1313 (size=12001) 2024-12-06T14:43:15,569 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/f560eb546ae5419580509e478791bfa3 2024-12-06T14:43:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/73182c8e24a5479da06a6efd6f745b25 is 50, key is test_row_0/C:col10/1733496194976/Put/seqid=0 2024-12-06T14:43:15,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742138_1314 (size=12001) 2024-12-06T14:43:15,592 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/73182c8e24a5479da06a6efd6f745b25 2024-12-06T14:43:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ba2346b3df994a4eae5ae6139aa2d953 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953 2024-12-06T14:43:15,602 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953, entries=150, sequenceid=49, filesize=30.2 K 2024-12-06T14:43:15,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/f560eb546ae5419580509e478791bfa3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f560eb546ae5419580509e478791bfa3 2024-12-06T14:43:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,608 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f560eb546ae5419580509e478791bfa3, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T14:43:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/73182c8e24a5479da06a6efd6f745b25 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73182c8e24a5479da06a6efd6f745b25 2024-12-06T14:43:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,614 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73182c8e24a5479da06a6efd6f745b25, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T14:43:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,614 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=0 B/0 for 415d8e5bcca7a5e3eb923a4249401464 in 96ms, sequenceid=49, compaction requested=false 2024-12-06T14:43:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-06T14:43:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-06T14:43:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-06T14:43:15,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 714 msec 2024-12-06T14:43:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 725 msec 2024-12-06T14:43:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:15,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206911500d32a794914a39db173de5051ca_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742139_1315 (size=14594) 2024-12-06T14:43:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,756 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206911500d32a794914a39db173de5051ca_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206911500d32a794914a39db173de5051ca_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/03f85eead85a436aaca888dc4193c5bf, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/03f85eead85a436aaca888dc4193c5bf is 175, key is test_row_0/A:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742140_1316 (size=39549) 2024-12-06T14:43:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,772 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/03f85eead85a436aaca888dc4193c5bf 2024-12-06T14:43:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/93ce57d6abb44c39a115743300bf40a9 is 50, key is test_row_0/B:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:15,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742141_1317 (size=14341) 2024-12-06T14:43:15,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496195782, max timestamp=1733496195812Num uniques 859; 2024-12-06T14:43:15,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496195782, max timestamp=1733496195812Num uniques 859; 2024-12-06T14:43:15,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733496195782, max timestamp=1733496195812Num uniques 859; 2024-12-06T14:43:15,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:43:15,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:43:15,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T14:43:15,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496255832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496255833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496255834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496255842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496255845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496255948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496255958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496255958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496255959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496255968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T14:43:16,039 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:16,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-06T14:43:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T14:43:16,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:16,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:16,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T14:43:16,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496256168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496256168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496256168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496256168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496256178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/93ce57d6abb44c39a115743300bf40a9 2024-12-06T14:43:16,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/a33e6a24022345e9a5f7f2cea2a90d83 is 50, key is test_row_0/C:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:16,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T14:43:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:16,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742142_1318 (size=12001) 2024-12-06T14:43:16,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/a33e6a24022345e9a5f7f2cea2a90d83 2024-12-06T14:43:16,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/03f85eead85a436aaca888dc4193c5bf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf 2024-12-06T14:43:16,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf, entries=200, sequenceid=59, filesize=38.6 K 2024-12-06T14:43:16,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/93ce57d6abb44c39a115743300bf40a9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/93ce57d6abb44c39a115743300bf40a9 2024-12-06T14:43:16,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/93ce57d6abb44c39a115743300bf40a9, entries=200, sequenceid=59, filesize=14.0 K 2024-12-06T14:43:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/a33e6a24022345e9a5f7f2cea2a90d83 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/a33e6a24022345e9a5f7f2cea2a90d83 2024-12-06T14:43:16,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/a33e6a24022345e9a5f7f2cea2a90d83, entries=150, sequenceid=59, filesize=11.7 K 2024-12-06T14:43:16,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 415d8e5bcca7a5e3eb923a4249401464 in 478ms, sequenceid=59, compaction requested=true 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:16,220 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:16,220 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:16,221 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,221 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,221 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/2cf9d31c3d934e9cac724e1fb9e5fed4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f560eb546ae5419580509e478791bfa3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/93ce57d6abb44c39a115743300bf40a9] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=37.4 K 2024-12-06T14:43:16,221 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=115.9 K 2024-12-06T14:43:16,221 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf] 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf9d31c3d934e9cac724e1fb9e5fed4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733496194880 2024-12-06T14:43:16,221 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a11efba1a3c4a4eb408df8f9c5b7004, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733496194880 2024-12-06T14:43:16,222 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f560eb546ae5419580509e478791bfa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733496194966 2024-12-06T14:43:16,222 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba2346b3df994a4eae5ae6139aa2d953, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733496194966 2024-12-06T14:43:16,222 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ce57d6abb44c39a115743300bf40a9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733496195702 2024-12-06T14:43:16,222 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03f85eead85a436aaca888dc4193c5bf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733496195702 2024-12-06T14:43:16,229 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:16,231 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#274 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:16,231 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/a78b651ea41145539fee4f0419ff3bd5 is 50, key is test_row_0/B:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:16,237 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206304eb56bee3941d783c52dc9242ca9eb_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:16,238 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206304eb56bee3941d783c52dc9242ca9eb_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:16,238 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206304eb56bee3941d783c52dc9242ca9eb_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:16,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742143_1319 (size=12104) 2024-12-06T14:43:16,251 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/a78b651ea41145539fee4f0419ff3bd5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a78b651ea41145539fee4f0419ff3bd5 2024-12-06T14:43:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742144_1320 (size=4469) 2024-12-06T14:43:16,254 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#273 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:16,255 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/9c94431a11284a8ca7279c016c10c116 is 175, key is test_row_0/A:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:16,258 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into a78b651ea41145539fee4f0419ff3bd5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:16,258 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:16,258 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=13, startTime=1733496196220; duration=0sec 2024-12-06T14:43:16,258 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:16,258 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:16,258 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:16,259 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:16,259 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:16,259 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,259 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/ea2e46c1750547e385135b47b432a618, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73182c8e24a5479da06a6efd6f745b25, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/a33e6a24022345e9a5f7f2cea2a90d83] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=35.2 K 2024-12-06T14:43:16,260 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ea2e46c1750547e385135b47b432a618, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733496194880 2024-12-06T14:43:16,260 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 73182c8e24a5479da06a6efd6f745b25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733496194966 2024-12-06T14:43:16,260 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a33e6a24022345e9a5f7f2cea2a90d83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733496195702 2024-12-06T14:43:16,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742145_1321 (size=31058) 2024-12-06T14:43:16,266 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:16,266 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/e1e006d7a3114e949114ccaa830b793c is 50, key is test_row_0/C:col10/1733496195702/Put/seqid=0 2024-12-06T14:43:16,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742146_1322 (size=12104) 2024-12-06T14:43:16,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T14:43:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:16,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-06T14:43:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T14:43:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066576d2f1cb3743ff80e720d20697e0b4_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496195830/Put/seqid=0 2024-12-06T14:43:16,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742147_1323 (size=12154) 2024-12-06T14:43:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:16,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496256488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496256496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496256496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496256496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496256497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496256608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496256608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496256608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496256608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,666 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/9c94431a11284a8ca7279c016c10c116 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9c94431a11284a8ca7279c016c10c116 2024-12-06T14:43:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T14:43:16,670 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 9c94431a11284a8ca7279c016c10c116(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:16,670 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:16,670 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=13, startTime=1733496196220; duration=0sec 2024-12-06T14:43:16,671 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:16,671 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:16,675 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/e1e006d7a3114e949114ccaa830b793c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e1e006d7a3114e949114ccaa830b793c 2024-12-06T14:43:16,683 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into e1e006d7a3114e949114ccaa830b793c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:16,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:16,683 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=13, startTime=1733496196220; duration=0sec 2024-12-06T14:43:16,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:16,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:16,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:16,766 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066576d2f1cb3743ff80e720d20697e0b4_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066576d2f1cb3743ff80e720d20697e0b4_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:16,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/47696a3d8c7e47bc9c93163d016ee70c, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/47696a3d8c7e47bc9c93163d016ee70c is 175, key is test_row_0/A:col10/1733496195830/Put/seqid=0 2024-12-06T14:43:16,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742148_1324 (size=30955) 2024-12-06T14:43:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496256818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496256818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496256818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496256818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496257008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,035 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:43:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496257128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496257128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496257128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496257128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,171 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/47696a3d8c7e47bc9c93163d016ee70c 2024-12-06T14:43:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T14:43:17,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/8f4a25c939a241b2bf5f133fee38824c is 50, key is test_row_0/B:col10/1733496195830/Put/seqid=0 2024-12-06T14:43:17,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742149_1325 (size=12001) 2024-12-06T14:43:17,582 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/8f4a25c939a241b2bf5f133fee38824c 2024-12-06T14:43:17,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/28217449ec624cd993a1407c135d71e7 is 50, key is test_row_0/C:col10/1733496195830/Put/seqid=0 2024-12-06T14:43:17,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742150_1326 (size=12001) 2024-12-06T14:43:17,596 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/28217449ec624cd993a1407c135d71e7 2024-12-06T14:43:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/47696a3d8c7e47bc9c93163d016ee70c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c 2024-12-06T14:43:17,604 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c, entries=150, sequenceid=95, filesize=30.2 K 2024-12-06T14:43:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/8f4a25c939a241b2bf5f133fee38824c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8f4a25c939a241b2bf5f133fee38824c 2024-12-06T14:43:17,609 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8f4a25c939a241b2bf5f133fee38824c, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T14:43:17,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/28217449ec624cd993a1407c135d71e7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/28217449ec624cd993a1407c135d71e7 2024-12-06T14:43:17,614 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/28217449ec624cd993a1407c135d71e7, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T14:43:17,615 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 415d8e5bcca7a5e3eb923a4249401464 in 1263ms, sequenceid=95, compaction requested=false 2024-12-06T14:43:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-06T14:43:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-06T14:43:17,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-06T14:43:17,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-12-06T14:43:17,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.5810 sec 2024-12-06T14:43:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:17,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T14:43:17,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:17,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:17,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:17,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:17,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206adc91ca3b46a4091a882c569b74419e7_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:17,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742151_1327 (size=19474) 2024-12-06T14:43:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496257682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496257683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496257685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496257688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496257798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496257798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496257798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496257798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496258008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496258008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496258008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496258008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496258027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,032 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 415d8e5bcca7a5e3eb923a4249401464 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:18,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206adc91ca3b46a4091a882c569b74419e7_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206adc91ca3b46a4091a882c569b74419e7_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:18,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/774c9db3c36a4698b18033237a87beba, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:18,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/774c9db3c36a4698b18033237a87beba is 175, key is test_row_0/A:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:18,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742152_1328 (size=56733) 2024-12-06T14:43:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T14:43:18,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:18,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-06T14:43:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:18,193 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:18,194 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:18,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496258319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496258319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496258319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496258328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-06T14:43:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:18,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/774c9db3c36a4698b18033237a87beba 2024-12-06T14:43:18,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/540ac2bad3914baaae619dc3eb584751 is 50, key is test_row_0/B:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:18,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-06T14:43:18,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:18,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:18,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:18,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742153_1329 (size=12001) 2024-12-06T14:43:18,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/540ac2bad3914baaae619dc3eb584751 2024-12-06T14:43:18,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/5e47fbe26af94972a738c203797dd048 is 50, key is test_row_0/C:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:18,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742154_1330 (size=12001) 2024-12-06T14:43:18,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/5e47fbe26af94972a738c203797dd048 2024-12-06T14:43:18,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/774c9db3c36a4698b18033237a87beba as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba 2024-12-06T14:43:18,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba, entries=300, sequenceid=111, filesize=55.4 K 2024-12-06T14:43:18,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/540ac2bad3914baaae619dc3eb584751 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/540ac2bad3914baaae619dc3eb584751 2024-12-06T14:43:18,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/540ac2bad3914baaae619dc3eb584751, entries=150, sequenceid=111, filesize=11.7 K 2024-12-06T14:43:18,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/5e47fbe26af94972a738c203797dd048 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5e47fbe26af94972a738c203797dd048 2024-12-06T14:43:18,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5e47fbe26af94972a738c203797dd048, entries=150, sequenceid=111, filesize=11.7 K 2024-12-06T14:43:18,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 415d8e5bcca7a5e3eb923a4249401464 in 908ms, sequenceid=111, compaction requested=true 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:18,561 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:18,561 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:18,563 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:18,563 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:18,563 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,563 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a78b651ea41145539fee4f0419ff3bd5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8f4a25c939a241b2bf5f133fee38824c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/540ac2bad3914baaae619dc3eb584751] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=35.3 K 2024-12-06T14:43:18,563 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:18,563 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:18,563 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,563 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9c94431a11284a8ca7279c016c10c116, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=116.0 K 2024-12-06T14:43:18,564 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,564 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9c94431a11284a8ca7279c016c10c116, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba] 2024-12-06T14:43:18,564 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a78b651ea41145539fee4f0419ff3bd5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733496195702 2024-12-06T14:43:18,564 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f4a25c939a241b2bf5f133fee38824c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733496195830 2024-12-06T14:43:18,564 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c94431a11284a8ca7279c016c10c116, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733496195702 2024-12-06T14:43:18,565 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 540ac2bad3914baaae619dc3eb584751, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733496197652 2024-12-06T14:43:18,565 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47696a3d8c7e47bc9c93163d016ee70c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733496195830 2024-12-06T14:43:18,565 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774c9db3c36a4698b18033237a87beba, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733496196495 2024-12-06T14:43:18,570 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:18,577 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206f8679f8e65b540688cd3d725f9a7cc6e_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:18,579 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206f8679f8e65b540688cd3d725f9a7cc6e_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:18,580 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f8679f8e65b540688cd3d725f9a7cc6e_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:18,597 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:18,598 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/cd21a6707d1d4b35b4bc673b26d15c7e is 50, key is test_row_0/B:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742155_1331 (size=4469) 2024-12-06T14:43:18,622 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#282 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:18,623 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/7ea1f633a1dd4e719ddf34a453ec00bb is 175, key is test_row_0/A:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:18,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742156_1332 (size=12207) 2024-12-06T14:43:18,642 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/cd21a6707d1d4b35b4bc673b26d15c7e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/cd21a6707d1d4b35b4bc673b26d15c7e 2024-12-06T14:43:18,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742157_1333 (size=31161) 2024-12-06T14:43:18,650 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into cd21a6707d1d4b35b4bc673b26d15c7e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:18,650 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:18,650 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=13, startTime=1733496198561; duration=0sec 2024-12-06T14:43:18,650 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:18,650 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:18,650 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:18,650 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:18,651 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:18,651 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/7ea1f633a1dd4e719ddf34a453ec00bb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7ea1f633a1dd4e719ddf34a453ec00bb 2024-12-06T14:43:18,651 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,651 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e1e006d7a3114e949114ccaa830b793c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/28217449ec624cd993a1407c135d71e7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5e47fbe26af94972a738c203797dd048] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=35.3 K 2024-12-06T14:43:18,651 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e006d7a3114e949114ccaa830b793c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733496195702 2024-12-06T14:43:18,651 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 28217449ec624cd993a1407c135d71e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733496195830 2024-12-06T14:43:18,652 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e47fbe26af94972a738c203797dd048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733496197652 2024-12-06T14:43:18,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-06T14:43:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:18,654 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:43:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:18,658 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 7ea1f633a1dd4e719ddf34a453ec00bb(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:18,658 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:18,658 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=13, startTime=1733496198561; duration=0sec 2024-12-06T14:43:18,658 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:18,658 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120686833b12c83244ea9217fea754d55215_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496197687/Put/seqid=0 2024-12-06T14:43:18,672 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:18,672 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/2852965ed1cc410bba4d41c2544c89b3 is 50, key is test_row_0/C:col10/1733496197652/Put/seqid=0 2024-12-06T14:43:18,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742159_1335 (size=12207) 2024-12-06T14:43:18,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742158_1334 (size=12154) 2024-12-06T14:43:18,711 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/2852965ed1cc410bba4d41c2544c89b3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/2852965ed1cc410bba4d41c2544c89b3 2024-12-06T14:43:18,715 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into 2852965ed1cc410bba4d41c2544c89b3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:18,716 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:18,716 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=13, startTime=1733496198561; duration=0sec 2024-12-06T14:43:18,716 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:18,716 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:18,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:18,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496258853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496258854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496258859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496258861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496258968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496258968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496258978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496258978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:19,152 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120686833b12c83244ea9217fea754d55215_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120686833b12c83244ea9217fea754d55215_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6d5ee9b2b4b44423bf5733b68d0deb04, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:19,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6d5ee9b2b4b44423bf5733b68d0deb04 is 175, key is test_row_0/A:col10/1733496197687/Put/seqid=0 2024-12-06T14:43:19,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742160_1336 (size=30955) 2024-12-06T14:43:19,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496259178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496259188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496259198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496259199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:19,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496259488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496259498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496259508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496259508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:19,569 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6d5ee9b2b4b44423bf5733b68d0deb04 2024-12-06T14:43:19,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dab80056cf294d388029bb2dab7c14fa is 50, key is test_row_0/B:col10/1733496197687/Put/seqid=0 2024-12-06T14:43:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742161_1337 (size=12001) 2024-12-06T14:43:19,980 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dab80056cf294d388029bb2dab7c14fa 2024-12-06T14:43:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/f7f58728136b4dad9cf81bcc2353f0b6 is 50, key is test_row_0/C:col10/1733496197687/Put/seqid=0 2024-12-06T14:43:19,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742162_1338 (size=12001) 2024-12-06T14:43:20,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496259998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:20,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496260008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:20,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496260018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496260018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:20,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496260058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:20,062 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 415d8e5bcca7a5e3eb923a4249401464 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:20,390 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/f7f58728136b4dad9cf81bcc2353f0b6 2024-12-06T14:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6d5ee9b2b4b44423bf5733b68d0deb04 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04 2024-12-06T14:43:20,399 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04, entries=150, sequenceid=130, filesize=30.2 K 2024-12-06T14:43:20,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dab80056cf294d388029bb2dab7c14fa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dab80056cf294d388029bb2dab7c14fa 2024-12-06T14:43:20,404 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dab80056cf294d388029bb2dab7c14fa, entries=150, sequenceid=130, filesize=11.7 K 2024-12-06T14:43:20,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/f7f58728136b4dad9cf81bcc2353f0b6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f7f58728136b4dad9cf81bcc2353f0b6 2024-12-06T14:43:20,409 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f7f58728136b4dad9cf81bcc2353f0b6, entries=150, sequenceid=130, filesize=11.7 K 2024-12-06T14:43:20,410 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 415d8e5bcca7a5e3eb923a4249401464 in 1757ms, sequenceid=130, compaction requested=false 2024-12-06T14:43:20,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:20,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:20,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-06T14:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-06T14:43:20,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-06T14:43:20,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-12-06T14:43:20,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.2250 sec 2024-12-06T14:43:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:21,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:43:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:21,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206973cb894b8b24dd194c7a11d5949e80d_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:21,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742163_1339 (size=14794) 2024-12-06T14:43:21,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:21,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206973cb894b8b24dd194c7a11d5949e80d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206973cb894b8b24dd194c7a11d5949e80d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:21,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/408da551bef84f4f84a3a549bf75e052, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/408da551bef84f4f84a3a549bf75e052 is 175, key is test_row_0/A:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:21,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742164_1340 (size=39749) 2024-12-06T14:43:21,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496261047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496261048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496261049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496261052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496261158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496261168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496261168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496261169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496261368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496261378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496261378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496261378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/408da551bef84f4f84a3a549bf75e052 2024-12-06T14:43:21,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dfb7b0ea8467466dba992240cfc48cb8 is 50, key is test_row_0/B:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742165_1341 (size=12151) 2024-12-06T14:43:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496261678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496261688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496261688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496261688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:21,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dfb7b0ea8467466dba992240cfc48cb8 2024-12-06T14:43:21,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/f4d696b2c2f44a66bd0b7156a66fb1e3 is 50, key is test_row_0/C:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:21,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742166_1342 (size=12151) 2024-12-06T14:43:21,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/f4d696b2c2f44a66bd0b7156a66fb1e3 2024-12-06T14:43:21,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/408da551bef84f4f84a3a549bf75e052 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052 2024-12-06T14:43:21,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052, entries=200, sequenceid=147, filesize=38.8 K 2024-12-06T14:43:21,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dfb7b0ea8467466dba992240cfc48cb8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dfb7b0ea8467466dba992240cfc48cb8 2024-12-06T14:43:21,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dfb7b0ea8467466dba992240cfc48cb8, entries=150, sequenceid=147, filesize=11.9 K 2024-12-06T14:43:21,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/f4d696b2c2f44a66bd0b7156a66fb1e3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f4d696b2c2f44a66bd0b7156a66fb1e3 2024-12-06T14:43:21,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f4d696b2c2f44a66bd0b7156a66fb1e3, entries=150, sequenceid=147, filesize=11.9 K 2024-12-06T14:43:21,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 415d8e5bcca7a5e3eb923a4249401464 in 872ms, sequenceid=147, compaction requested=true 2024-12-06T14:43:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:21,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:21,883 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:21,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:21,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:21,884 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:21,884 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:21,884 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:21,884 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7ea1f633a1dd4e719ddf34a453ec00bb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=99.5 K 2024-12-06T14:43:21,885 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:21,885 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7ea1f633a1dd4e719ddf34a453ec00bb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052] 2024-12-06T14:43:21,885 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:21,885 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:21,885 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:21,885 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/cd21a6707d1d4b35b4bc673b26d15c7e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dab80056cf294d388029bb2dab7c14fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dfb7b0ea8467466dba992240cfc48cb8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=35.5 K 2024-12-06T14:43:21,885 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea1f633a1dd4e719ddf34a453ec00bb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733496197652 2024-12-06T14:43:21,885 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cd21a6707d1d4b35b4bc673b26d15c7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733496197652 2024-12-06T14:43:21,885 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5ee9b2b4b44423bf5733b68d0deb04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733496197681 2024-12-06T14:43:21,886 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting dab80056cf294d388029bb2dab7c14fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733496197681 2024-12-06T14:43:21,886 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408da551bef84f4f84a3a549bf75e052, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733496198853 2024-12-06T14:43:21,886 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb7b0ea8467466dba992240cfc48cb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733496198853 2024-12-06T14:43:21,895 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:21,896 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/6bba23e7a8314d6992eaeb85a5f7dab2 is 50, key is test_row_0/B:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:21,896 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:21,900 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206117f909f1ec34eafbd63e1febecb63e8_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:21,902 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206117f909f1ec34eafbd63e1febecb63e8_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:21,902 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206117f909f1ec34eafbd63e1febecb63e8_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:21,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742167_1343 (size=12459) 2024-12-06T14:43:21,917 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/6bba23e7a8314d6992eaeb85a5f7dab2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6bba23e7a8314d6992eaeb85a5f7dab2 2024-12-06T14:43:21,921 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into 6bba23e7a8314d6992eaeb85a5f7dab2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:21,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:21,922 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=13, startTime=1733496201883; duration=0sec 2024-12-06T14:43:21,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:21,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:21,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:21,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:21,922 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:21,923 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:21,923 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/2852965ed1cc410bba4d41c2544c89b3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f7f58728136b4dad9cf81bcc2353f0b6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f4d696b2c2f44a66bd0b7156a66fb1e3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=35.5 K 2024-12-06T14:43:21,923 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 2852965ed1cc410bba4d41c2544c89b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733496197652 2024-12-06T14:43:21,923 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f58728136b4dad9cf81bcc2353f0b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733496197681 2024-12-06T14:43:21,923 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d696b2c2f44a66bd0b7156a66fb1e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733496198853 2024-12-06T14:43:21,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742168_1344 (size=4469) 2024-12-06T14:43:21,934 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:21,935 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/c35e2c889e4e49fa891dbd892f55e4e6 is 50, key is test_row_0/C:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:21,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742169_1345 (size=12459) 2024-12-06T14:43:21,952 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/c35e2c889e4e49fa891dbd892f55e4e6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/c35e2c889e4e49fa891dbd892f55e4e6 2024-12-06T14:43:21,963 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into c35e2c889e4e49fa891dbd892f55e4e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:21,963 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:21,963 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=13, startTime=1733496201884; duration=0sec 2024-12-06T14:43:21,963 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:21,963 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:22,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:43:22,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:22,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:22,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:22,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:22,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:22,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:22,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066a0ee7f644444e859051fc0f51550b41_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496201051/Put/seqid=0 2024-12-06T14:43:22,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742170_1346 (size=14794) 2024-12-06T14:43:22,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496262216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496262220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496262228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496262231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,327 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#292 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:22,327 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/43e4690a04ca45a581604ed81446cfeb is 175, key is test_row_0/A:col10/1733496201010/Put/seqid=0 2024-12-06T14:43:22,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742171_1347 (size=31413) 2024-12-06T14:43:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496262338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496262338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496262348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496262348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T14:43:22,359 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:22,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-06T14:43:22,363 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T14:43:22,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:22,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T14:43:22,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T14:43:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:22,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496262549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496262549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496262558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496262558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:22,621 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066a0ee7f644444e859051fc0f51550b41_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066a0ee7f644444e859051fc0f51550b41_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:22,623 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ad31f2cd8c09484981b1a02edb9369aa, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:22,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ad31f2cd8c09484981b1a02edb9369aa is 175, key is test_row_0/A:col10/1733496201051/Put/seqid=0 2024-12-06T14:43:22,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742172_1348 (size=39749) 2024-12-06T14:43:22,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ad31f2cd8c09484981b1a02edb9369aa 2024-12-06T14:43:22,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dce22544f47e44869b1eb22952d6494d is 50, key is test_row_0/B:col10/1733496201051/Put/seqid=0 2024-12-06T14:43:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742173_1349 (size=12151) 2024-12-06T14:43:22,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dce22544f47e44869b1eb22952d6494d 2024-12-06T14:43:22,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T14:43:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:22,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:22,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/35a28c5a97f4419b8c9e1aa2af9f2a40 is 50, key is test_row_0/C:col10/1733496201051/Put/seqid=0 2024-12-06T14:43:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T14:43:22,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742174_1350 (size=12151) 2024-12-06T14:43:22,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/35a28c5a97f4419b8c9e1aa2af9f2a40 2024-12-06T14:43:22,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/ad31f2cd8c09484981b1a02edb9369aa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa 2024-12-06T14:43:22,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa, entries=200, sequenceid=167, filesize=38.8 K 2024-12-06T14:43:22,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/dce22544f47e44869b1eb22952d6494d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dce22544f47e44869b1eb22952d6494d 2024-12-06T14:43:22,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dce22544f47e44869b1eb22952d6494d, entries=150, sequenceid=167, filesize=11.9 K 2024-12-06T14:43:22,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/35a28c5a97f4419b8c9e1aa2af9f2a40 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/35a28c5a97f4419b8c9e1aa2af9f2a40 2024-12-06T14:43:22,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/35a28c5a97f4419b8c9e1aa2af9f2a40, entries=150, sequenceid=167, filesize=11.9 K 2024-12-06T14:43:22,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 415d8e5bcca7a5e3eb923a4249401464 in 529ms, sequenceid=167, compaction requested=false 2024-12-06T14:43:22,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:22,737 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/43e4690a04ca45a581604ed81446cfeb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/43e4690a04ca45a581604ed81446cfeb 2024-12-06T14:43:22,747 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 43e4690a04ca45a581604ed81446cfeb(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:22,747 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:22,747 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=13, startTime=1733496201883; duration=0sec 2024-12-06T14:43:22,747 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:22,747 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:22,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T14:43:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:22,824 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T14:43:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060241b6b7a2544e0b96576a7babb961b3_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496202227/Put/seqid=0 2024-12-06T14:43:22,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742175_1351 (size=12304) 2024-12-06T14:43:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:22,842 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060241b6b7a2544e0b96576a7babb961b3_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060241b6b7a2544e0b96576a7babb961b3_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:22,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/329a88f240894966ab1e4ea314390510, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/329a88f240894966ab1e4ea314390510 is 175, key is test_row_0/A:col10/1733496202227/Put/seqid=0 2024-12-06T14:43:22,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742176_1352 (size=31105) 2024-12-06T14:43:22,849 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/329a88f240894966ab1e4ea314390510 2024-12-06T14:43:22,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/3e25cdb8553949dabd9784194c03a74b is 50, key is test_row_0/B:col10/1733496202227/Put/seqid=0 2024-12-06T14:43:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:22,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:22,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742177_1353 (size=12151) 2024-12-06T14:43:22,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496262901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496262902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496262902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496262903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T14:43:23,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496263008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496263018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496263018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496263018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496263219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496263228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496263228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496263228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,275 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/3e25cdb8553949dabd9784194c03a74b 2024-12-06T14:43:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/4e3f2fa2df3c46368483cffa4d74fd97 is 50, key is test_row_0/C:col10/1733496202227/Put/seqid=0 2024-12-06T14:43:23,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742178_1354 (size=12151) 2024-12-06T14:43:23,289 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/4e3f2fa2df3c46368483cffa4d74fd97 2024-12-06T14:43:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/329a88f240894966ab1e4ea314390510 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510 2024-12-06T14:43:23,296 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510, entries=150, sequenceid=182, filesize=30.4 K 2024-12-06T14:43:23,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/3e25cdb8553949dabd9784194c03a74b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/3e25cdb8553949dabd9784194c03a74b 2024-12-06T14:43:23,302 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/3e25cdb8553949dabd9784194c03a74b, entries=150, sequenceid=182, filesize=11.9 K 2024-12-06T14:43:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/4e3f2fa2df3c46368483cffa4d74fd97 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/4e3f2fa2df3c46368483cffa4d74fd97 2024-12-06T14:43:23,307 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/4e3f2fa2df3c46368483cffa4d74fd97, entries=150, sequenceid=182, filesize=11.9 K 2024-12-06T14:43:23,307 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 415d8e5bcca7a5e3eb923a4249401464 in 483ms, sequenceid=182, compaction requested=true 2024-12-06T14:43:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-06T14:43:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-06T14:43:23,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-06T14:43:23,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 945 msec 2024-12-06T14:43:23,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 951 msec 2024-12-06T14:43:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T14:43:23,498 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:23,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-06T14:43:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T14:43:23,500 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:23,501 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:23,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:23,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:43:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:23,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206513caca63a6f44d393bbfa2b51b74f98_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:23,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742179_1355 (size=17284) 2024-12-06T14:43:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496263555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496263556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496263564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496263565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T14:43:23,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:23,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:23,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496263668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496263668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496263678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496263679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:23,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:23,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T14:43:23,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496263878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496263878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496263889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496263889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:23,952 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:23,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206513caca63a6f44d393bbfa2b51b74f98_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206513caca63a6f44d393bbfa2b51b74f98_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:23,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/f89ffe1382b14870866b7aeda2c17fe4, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:23,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/f89ffe1382b14870866b7aeda2c17fe4 is 175, key is test_row_0/A:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:23,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742180_1356 (size=48389) 2024-12-06T14:43:24,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496264108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,115 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 415d8e5bcca7a5e3eb923a4249401464 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T14:43:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496264188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496264198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496264208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496264208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/f89ffe1382b14870866b7aeda2c17fe4 2024-12-06T14:43:24,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/bef6f0b5c7c04f559d654769d0dfcc29 is 50, key is test_row_0/B:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:24,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742181_1357 (size=12151) 2024-12-06T14:43:24,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/bef6f0b5c7c04f559d654769d0dfcc29 2024-12-06T14:43:24,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/7682cecbf65d4229bdd0471944b69656 is 50, key is test_row_0/C:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:24,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742182_1358 (size=12151) 2024-12-06T14:43:24,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/7682cecbf65d4229bdd0471944b69656 2024-12-06T14:43:24,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:24,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/f89ffe1382b14870866b7aeda2c17fe4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4 2024-12-06T14:43:24,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4, entries=250, sequenceid=202, filesize=47.3 K 2024-12-06T14:43:24,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/bef6f0b5c7c04f559d654769d0dfcc29 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bef6f0b5c7c04f559d654769d0dfcc29 2024-12-06T14:43:24,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bef6f0b5c7c04f559d654769d0dfcc29, entries=150, sequenceid=202, filesize=11.9 K 2024-12-06T14:43:24,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/7682cecbf65d4229bdd0471944b69656 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/7682cecbf65d4229bdd0471944b69656 2024-12-06T14:43:24,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/7682cecbf65d4229bdd0471944b69656, entries=150, sequenceid=202, filesize=11.9 K 2024-12-06T14:43:24,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 415d8e5bcca7a5e3eb923a4249401464 in 923ms, sequenceid=202, compaction requested=true 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:24,465 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:24,465 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:24,466 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:24,466 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:24,466 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,466 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/43e4690a04ca45a581604ed81446cfeb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=147.1 K 2024-12-06T14:43:24,467 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,467 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/43e4690a04ca45a581604ed81446cfeb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4] 2024-12-06T14:43:24,467 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:24,467 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:24,467 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e4690a04ca45a581604ed81446cfeb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733496198853 2024-12-06T14:43:24,467 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,467 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6bba23e7a8314d6992eaeb85a5f7dab2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dce22544f47e44869b1eb22952d6494d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/3e25cdb8553949dabd9784194c03a74b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bef6f0b5c7c04f559d654769d0dfcc29] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=47.8 K 2024-12-06T14:43:24,467 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bba23e7a8314d6992eaeb85a5f7dab2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733496198853 2024-12-06T14:43:24,467 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad31f2cd8c09484981b1a02edb9369aa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733496201046 2024-12-06T14:43:24,468 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting dce22544f47e44869b1eb22952d6494d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733496201047 2024-12-06T14:43:24,468 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 329a88f240894966ab1e4ea314390510, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733496202218 2024-12-06T14:43:24,468 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e25cdb8553949dabd9784194c03a74b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733496202218 2024-12-06T14:43:24,468 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f89ffe1382b14870866b7aeda2c17fe4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733496202900 2024-12-06T14:43:24,469 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting bef6f0b5c7c04f559d654769d0dfcc29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733496202902 2024-12-06T14:43:24,479 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:24,480 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/72f1d41ee0b64139b3979e6a5c2e4460 is 50, key is test_row_0/B:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:24,481 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:24,483 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412064a58349752014fd0ab82bd942da97f72_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:24,485 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412064a58349752014fd0ab82bd942da97f72_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:24,485 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064a58349752014fd0ab82bd942da97f72_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:24,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742184_1360 (size=4469) 2024-12-06T14:43:24,523 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#304 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:24,523 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/c7e15711e9a346eb800b33847017a8fa is 175, key is test_row_0/A:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:24,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742183_1359 (size=12595) 2024-12-06T14:43:24,541 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/72f1d41ee0b64139b3979e6a5c2e4460 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/72f1d41ee0b64139b3979e6a5c2e4460 2024-12-06T14:43:24,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742185_1361 (size=31549) 2024-12-06T14:43:24,551 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into 72f1d41ee0b64139b3979e6a5c2e4460(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:24,551 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:24,551 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=12, startTime=1733496204465; duration=0sec 2024-12-06T14:43:24,552 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:24,552 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:24,552 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:24,554 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:24,554 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:24,554 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,554 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/c35e2c889e4e49fa891dbd892f55e4e6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/35a28c5a97f4419b8c9e1aa2af9f2a40, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/4e3f2fa2df3c46368483cffa4d74fd97, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/7682cecbf65d4229bdd0471944b69656] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=47.8 K 2024-12-06T14:43:24,555 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c35e2c889e4e49fa891dbd892f55e4e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733496198853 2024-12-06T14:43:24,555 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 35a28c5a97f4419b8c9e1aa2af9f2a40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733496201047 2024-12-06T14:43:24,556 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e3f2fa2df3c46368483cffa4d74fd97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733496202218 2024-12-06T14:43:24,556 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 7682cecbf65d4229bdd0471944b69656, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733496202902 2024-12-06T14:43:24,559 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/c7e15711e9a346eb800b33847017a8fa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/c7e15711e9a346eb800b33847017a8fa 2024-12-06T14:43:24,568 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:24,568 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/d45f11064ec8425d80eebeb713fade67 is 50, key is test_row_0/C:col10/1733496203540/Put/seqid=0 2024-12-06T14:43:24,570 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into c7e15711e9a346eb800b33847017a8fa(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:24,570 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:24,570 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=12, startTime=1733496204465; duration=0sec 2024-12-06T14:43:24,570 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:24,570 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:24,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:24,571 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:24,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206357addcfd365484882f6f29a06fb8e6c_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496203554/Put/seqid=0 2024-12-06T14:43:24,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742186_1362 (size=12595) 2024-12-06T14:43:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T14:43:24,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742187_1363 (size=12304) 2024-12-06T14:43:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:24,647 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206357addcfd365484882f6f29a06fb8e6c_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206357addcfd365484882f6f29a06fb8e6c_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/8421951a7e524ff19c63229368693d3e, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/8421951a7e524ff19c63229368693d3e is 175, key is test_row_0/A:col10/1733496203554/Put/seqid=0 2024-12-06T14:43:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742188_1364 (size=31105) 2024-12-06T14:43:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:24,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:24,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496264745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496264750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496264751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496264752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496264858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496264868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496264868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496264869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,010 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/d45f11064ec8425d80eebeb713fade67 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/d45f11064ec8425d80eebeb713fade67 2024-12-06T14:43:25,015 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into d45f11064ec8425d80eebeb713fade67(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:25,015 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:25,015 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=12, startTime=1733496204465; duration=0sec 2024-12-06T14:43:25,015 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:25,015 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:25,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/8421951a7e524ff19c63229368693d3e 2024-12-06T14:43:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496265069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/40c0fb9119574267835688cfba642146 is 50, key is test_row_0/B:col10/1733496203554/Put/seqid=0 2024-12-06T14:43:25,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742189_1365 (size=12151) 2024-12-06T14:43:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496265078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,082 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/40c0fb9119574267835688cfba642146 2024-12-06T14:43:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496265078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:25,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496265079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/73a260e030eb43859c3dcce9145519fe is 50, key is test_row_0/C:col10/1733496203554/Put/seqid=0 2024-12-06T14:43:25,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742190_1366 (size=12151) 2024-12-06T14:43:25,114 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/73a260e030eb43859c3dcce9145519fe 2024-12-06T14:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/8421951a7e524ff19c63229368693d3e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e 2024-12-06T14:43:25,123 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e, entries=150, sequenceid=216, filesize=30.4 K 2024-12-06T14:43:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/40c0fb9119574267835688cfba642146 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/40c0fb9119574267835688cfba642146 2024-12-06T14:43:25,128 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/40c0fb9119574267835688cfba642146, entries=150, sequenceid=216, filesize=11.9 K 2024-12-06T14:43:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/73a260e030eb43859c3dcce9145519fe as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73a260e030eb43859c3dcce9145519fe 2024-12-06T14:43:25,134 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73a260e030eb43859c3dcce9145519fe, entries=150, sequenceid=216, filesize=11.9 K 2024-12-06T14:43:25,136 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 415d8e5bcca7a5e3eb923a4249401464 in 565ms, sequenceid=216, compaction requested=false 2024-12-06T14:43:25,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:25,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:25,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-06T14:43:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-06T14:43:25,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-06T14:43:25,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6370 sec 2024-12-06T14:43:25,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6440 sec 2024-12-06T14:43:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496204744, max timestamp=1733496205389Num uniques 859; 2024-12-06T14:43:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496204744, max timestamp=1733496205389Num uniques 859; 2024-12-06T14:43:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496204744, max timestamp=1733496205389Num uniques 859; 2024-12-06T14:43:25,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:25,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:25,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-12-06T14:43:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:25,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206de9715347296456e9804dd3a7d2a340d_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:25,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742191_1367 (size=14794) 2024-12-06T14:43:25,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496205425, max timestamp=1733496205442Num uniques 859; 2024-12-06T14:43:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496205425, max timestamp=1733496205442Num uniques 859; 2024-12-06T14:43:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496205425, max timestamp=1733496205442Num uniques 859; 2024-12-06T14:43:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:25,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:25,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206de9715347296456e9804dd3a7d2a340d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206de9715347296456e9804dd3a7d2a340d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:25,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/30181bf020d8430d83c49ae45b2fa1d5, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:25,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/30181bf020d8430d83c49ae45b2fa1d5 is 175, key is test_row_0/A:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:25,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742192_1368 (size=39749) 2024-12-06T14:43:25,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496205483, max timestamp=1733496205494Num uniques 1018; 2024-12-06T14:43:25,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:43:25,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496205483, max timestamp=1733496205494Num uniques 1018; 2024-12-06T14:43:25,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496205483, max timestamp=1733496205494Num uniques 1018; 2024-12-06T14:43:25,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:43:25,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:43:25,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733496205523, max timestamp=1733496205548Num uniques 1077; 2024-12-06T14:43:25,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-06T14:43:25,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733496205523, max timestamp=1733496205548Num uniques 1077; 2024-12-06T14:43:25,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-06T14:43:25,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733496205523, max timestamp=1733496205548Num uniques 1077; 2024-12-06T14:43:25,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-06T14:43:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733496205574, max timestamp=1733496205622Num uniques 1086; 2024-12-06T14:43:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-06T14:43:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733496205574, max timestamp=1733496205622Num uniques 1086; 2024-12-06T14:43:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733496205574, max timestamp=1733496205622Num uniques 1086; 2024-12-06T14:43:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-06T14:43:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-06T14:43:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T14:43:25,649 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:25,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-06T14:43:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:25,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:25,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:25,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733496205648, max timestamp=1733496205668Num uniques 1145; 2024-12-06T14:43:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-06T14:43:25,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733496205648, max timestamp=1733496205668Num uniques 1145; 2024-12-06T14:43:25,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-06T14:43:25,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733496205648, max timestamp=1733496205668Num uniques 1145; 2024-12-06T14:43:25,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-06T14:43:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496265700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733496205689, max timestamp=1733496205702Num uniques 1154; 2024-12-06T14:43:25,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-06T14:43:25,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733496205689, max timestamp=1733496205702Num uniques 1154; 2024-12-06T14:43:25,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-06T14:43:25,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733496205689, max timestamp=1733496205702Num uniques 1104; 2024-12-06T14:43:25,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-06T14:43:25,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733496205723, max timestamp=1733496205742Num uniques 1113; 2024-12-06T14:43:25,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-06T14:43:25,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733496205723, max timestamp=1733496205742Num uniques 1113; 2024-12-06T14:43:25,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-06T14:43:25,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733496205723, max timestamp=1733496205742Num uniques 1113; 2024-12-06T14:43:25,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-06T14:43:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:25,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:25,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:25,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7866, totalHeapSize=18128, min timestamp=1733496205765, max timestamp=1733496205803Num uniques 1122; 2024-12-06T14:43:25,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51376, new segment data size=7866, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-06T14:43:25,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7863, totalHeapSize=18128, min timestamp=1733496205765, max timestamp=1733496205803Num uniques 1122; 2024-12-06T14:43:25,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733496205765, max timestamp=1733496205803Num uniques 1122; 2024-12-06T14:43:25,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51373, new segment data size=7863, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-06T14:43:25,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-06T14:43:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:25,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1733496205831, max timestamp=1733496205845Num uniques 1031; 2024-12-06T14:43:25,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-12-06T14:43:25,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1733496205831, max timestamp=1733496205845Num uniques 1031; 2024-12-06T14:43:25,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-12-06T14:43:25,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1733496205831, max timestamp=1733496205845Num uniques 1031; 2024-12-06T14:43:25,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-12-06T14:43:25,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/30181bf020d8430d83c49ae45b2fa1d5 2024-12-06T14:43:25,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/6f2957719c264706b90b5c295b251811 is 50, key is test_row_0/B:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:25,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742193_1369 (size=12151) 2024-12-06T14:43:25,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733496205860, max timestamp=1733496205910Num uniques 1140; 2024-12-06T14:43:25,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=15570, suffix heap size=146528,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=340 2024-12-06T14:43:25,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733496205860, max timestamp=1733496205910Num uniques 1140; 2024-12-06T14:43:25,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=15570, suffix heap size=146528,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=340 2024-12-06T14:43:25,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733496205860, max timestamp=1733496205910Num uniques 1140; 2024-12-06T14:43:25,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=15570, suffix heap size=146528,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=340 2024-12-06T14:43:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496265946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:25,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:25,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733496205943, max timestamp=1733496205956Num uniques 1199; 2024-12-06T14:43:25,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:25,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=9114, suffix heap size=152664,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=199 2024-12-06T14:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:25,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733496205943, max timestamp=1733496205956Num uniques 1199; 2024-12-06T14:43:25,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=9114, suffix heap size=152664,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=199 2024-12-06T14:43:25,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733496205943, max timestamp=1733496205956Num uniques 1199; 2024-12-06T14:43:25,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=9114, suffix heap size=152664,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=199 2024-12-06T14:43:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:26,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733496206005, max timestamp=1733496206040Num uniques 1058; 2024-12-06T14:43:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-06T14:43:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733496206005, max timestamp=1733496206040Num uniques 1058; 2024-12-06T14:43:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-06T14:43:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733496206005, max timestamp=1733496206040Num uniques 1058; 2024-12-06T14:43:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-06T14:43:26,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733496206068, max timestamp=1733496206088Num uniques 1067; 2024-12-06T14:43:26,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733496206068, max timestamp=1733496206088Num uniques 1067; 2024-12-06T14:43:26,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=12222, suffix heap size=138936,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-12-06T14:43:26,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=12222, suffix heap size=138936,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-12-06T14:43:26,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733496206068, max timestamp=1733496206088Num uniques 1067; 2024-12-06T14:43:26,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=12222, suffix heap size=138936,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-12-06T14:43:26,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1733496206122, max timestamp=1733496206142Num uniques 1126; 2024-12-06T14:43:26,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=14926, suffix heap size=145072,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=326 2024-12-06T14:43:26,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1733496206122, max timestamp=1733496206142Num uniques 1126; 2024-12-06T14:43:26,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=14926, suffix heap size=145072,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=326 2024-12-06T14:43:26,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1733496206122, max timestamp=1733496206142Num uniques 1126; 2024-12-06T14:43:26,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=14926, suffix heap size=145072,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=326 2024-12-06T14:43:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733496206197, max timestamp=1733496206215Num uniques 1185; 2024-12-06T14:43:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733496206197, max timestamp=1733496206215Num uniques 1185; 2024-12-06T14:43:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=13050, suffix heap size=151208,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=285 2024-12-06T14:43:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=13050, suffix heap size=151208,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=285 2024-12-06T14:43:26,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733496206197, max timestamp=1733496206215Num uniques 1185; 2024-12-06T14:43:26,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=13050, suffix heap size=151208,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=285 2024-12-06T14:43:26,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:26,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/6f2957719c264706b90b5c295b251811 2024-12-06T14:43:26,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496266283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733496206259, max timestamp=1733496206295Num uniques 1144; 2024-12-06T14:43:26,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-06T14:43:26,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8874, totalHeapSize=20416, min timestamp=1733496206259, max timestamp=1733496206296Num uniques 1144; 2024-12-06T14:43:26,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52384, new segment data size=8874, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-06T14:43:26,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733496206259, max timestamp=1733496206296Num uniques 1144; 2024-12-06T14:43:26,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52384, new segment data size=8102, suffix heap size=146944,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=177 2024-12-06T14:43:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/fe0b88b1cc614ea586a94c30906a5117 is 50, key is test_row_0/C:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:26,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742194_1370 (size=12151) 2024-12-06T14:43:26,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733496206356, max timestamp=1733496206384Num uniques 1053; 2024-12-06T14:43:26,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-06T14:43:26,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496206356, max timestamp=1733496206384Num uniques 1053; 2024-12-06T14:43:26,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=6870, suffix heap size=137480,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=150 2024-12-06T14:43:26,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496206356, max timestamp=1733496206384Num uniques 1036; 2024-12-06T14:43:26,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47453, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-06T14:43:26,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733496206414, max timestamp=1733496206458Num uniques 1009; 2024-12-06T14:43:26,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7410, totalHeapSize=17088, min timestamp=1733496206414, max timestamp=1733496206458Num uniques 1009; 2024-12-06T14:43:26,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46205, new segment data size=7410, suffix heap size=132904,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=162 2024-12-06T14:43:26,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7410, totalHeapSize=17088, min timestamp=1733496206414, max timestamp=1733496206458Num uniques 1012; 2024-12-06T14:43:26,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=7412, suffix heap size=132904,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=162 2024-12-06T14:43:26,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46340, new segment data size=7410, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-06T14:43:26,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733496206485, max timestamp=1733496206514Num uniques 1021; 2024-12-06T14:43:26,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733496206485, max timestamp=1733496206514Num uniques 1021; 2024-12-06T14:43:26,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733496206485, max timestamp=1733496206514Num uniques 1021; 2024-12-06T14:43:26,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-06T14:43:26,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-06T14:43:26,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-06T14:43:26,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733496206537, max timestamp=1733496206550Num uniques 1030; 2024-12-06T14:43:26,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=12820, suffix heap size=135088,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=280 2024-12-06T14:43:26,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733496206537, max timestamp=1733496206550Num uniques 1030; 2024-12-06T14:43:26,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=12820, suffix heap size=135088,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=280 2024-12-06T14:43:26,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733496206537, max timestamp=1733496206550Num uniques 1030; 2024-12-06T14:43:26,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=12820, suffix heap size=135088,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=280 2024-12-06T14:43:26,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=339, cellSize=15524, totalHeapSize=35496, min timestamp=1733496206575, max timestamp=1733496206593Num uniques 1139; 2024-12-06T14:43:26,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=15524, suffix heap size=146424,new segment heap size=35496 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=339 2024-12-06T14:43:26,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=339, cellSize=15524, totalHeapSize=35496, min timestamp=1733496206575, max timestamp=1733496206593Num uniques 1139; 2024-12-06T14:43:26,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=15524, suffix heap size=146424,new segment heap size=35496 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=339 2024-12-06T14:43:26,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=339, cellSize=15524, totalHeapSize=35496, min timestamp=1733496206575, max timestamp=1733496206593Num uniques 1139; 2024-12-06T14:43:26,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=15524, suffix heap size=146424,new segment heap size=35496 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=339 2024-12-06T14:43:26,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496266624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=398, cellSize=18228, totalHeapSize=41632, min timestamp=1733496206612, max timestamp=1733496206631Num uniques 1198; 2024-12-06T14:43:26,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54868, new segment data size=18228, suffix heap size=152560,new segment heap size=41632 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1198, new segment cells count=398 2024-12-06T14:43:26,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=398, cellSize=18228, totalHeapSize=41632, min timestamp=1733496206612, max timestamp=1733496206631Num uniques 1198; 2024-12-06T14:43:26,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54868, new segment data size=18228, suffix heap size=152560,new segment heap size=41632 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1198, new segment cells count=398 2024-12-06T14:43:26,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733496206612, max timestamp=1733496206631Num uniques 1198; 2024-12-06T14:43:26,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54868, new segment data size=11358, suffix heap size=152560,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1198, new segment cells count=248 2024-12-06T14:43:26,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496266670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496266678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496266679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:26,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:26,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/fe0b88b1cc614ea586a94c30906a5117 2024-12-06T14:43:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/30181bf020d8430d83c49ae45b2fa1d5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5 2024-12-06T14:43:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496266738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5, entries=200, sequenceid=238, filesize=38.8 K 2024-12-06T14:43:26,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/6f2957719c264706b90b5c295b251811 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6f2957719c264706b90b5c295b251811 2024-12-06T14:43:26,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6f2957719c264706b90b5c295b251811, entries=150, sequenceid=238, filesize=11.9 K 2024-12-06T14:43:26,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/fe0b88b1cc614ea586a94c30906a5117 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fe0b88b1cc614ea586a94c30906a5117 2024-12-06T14:43:26,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fe0b88b1cc614ea586a94c30906a5117, entries=150, sequenceid=238, filesize=11.9 K 2024-12-06T14:43:26,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=161.02 KB/164880 for 415d8e5bcca7a5e3eb923a4249401464 in 1360ms, sequenceid=238, compaction requested=true 2024-12-06T14:43:26,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:26,755 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:26,755 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:26,757 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:26,757 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:26,758 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,758 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,758 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/72f1d41ee0b64139b3979e6a5c2e4460, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/40c0fb9119574267835688cfba642146, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6f2957719c264706b90b5c295b251811] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=36.0 K 2024-12-06T14:43:26,758 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/c7e15711e9a346eb800b33847017a8fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=100.0 K 2024-12-06T14:43:26,758 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/c7e15711e9a346eb800b33847017a8fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5] 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f1d41ee0b64139b3979e6a5c2e4460, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733496202902 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e15711e9a346eb800b33847017a8fa, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733496202902 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 40c0fb9119574267835688cfba642146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733496203551 2024-12-06T14:43:26,758 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8421951a7e524ff19c63229368693d3e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733496203551 2024-12-06T14:43:26,759 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30181bf020d8430d83c49ae45b2fa1d5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733496204744 2024-12-06T14:43:26,759 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2957719c264706b90b5c295b251811, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733496204744 2024-12-06T14:43:26,767 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:26,768 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:26,769 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120634bb990511dc4f64b9ae3856aa4f8c7d_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:26,769 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/9074a01d12314a43aaaf1e3cf528e8c4 is 50, key is test_row_0/B:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:26,771 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120634bb990511dc4f64b9ae3856aa4f8c7d_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:26,771 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120634bb990511dc4f64b9ae3856aa4f8c7d_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:26,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742195_1371 (size=12697) 2024-12-06T14:43:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742196_1372 (size=4469) 2024-12-06T14:43:26,777 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#313 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:26,778 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/1f72a205e50b4837a7f6bf1d7463c975 is 175, key is test_row_0/A:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:26,778 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/9074a01d12314a43aaaf1e3cf528e8c4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9074a01d12314a43aaaf1e3cf528e8c4 2024-12-06T14:43:26,784 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into 9074a01d12314a43aaaf1e3cf528e8c4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:26,784 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:26,784 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=13, startTime=1733496206755; duration=0sec 2024-12-06T14:43:26,784 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:26,784 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:26,784 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:26,787 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:26,787 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:26,787 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,787 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/d45f11064ec8425d80eebeb713fade67, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73a260e030eb43859c3dcce9145519fe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fe0b88b1cc614ea586a94c30906a5117] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=36.0 K 2024-12-06T14:43:26,787 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d45f11064ec8425d80eebeb713fade67, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733496202902 2024-12-06T14:43:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:26,788 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a260e030eb43859c3dcce9145519fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733496203551 2024-12-06T14:43:26,788 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0b88b1cc614ea586a94c30906a5117, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733496204744 2024-12-06T14:43:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:26,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=167.72 KB heapSize=467.58 KB 2024-12-06T14:43:26,798 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:26,799 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/fea1e4d480af4378bb9f865f396bf9c3 is 50, key is test_row_0/C:col10/1733496204750/Put/seqid=0 2024-12-06T14:43:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733496206660, max timestamp=1733496206797Num uniques 1107; 2024-12-06T14:43:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=7185, suffix heap size=143096,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=157 2024-12-06T14:43:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733496206660, max timestamp=1733496206797Num uniques 1257; 2024-12-06T14:43:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733496206660, max timestamp=1733496206797Num uniques 1257; 2024-12-06T14:43:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=57565, new segment data size=7185, suffix heap size=158696,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1257, new segment cells count=157 2024-12-06T14:43:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=57565, new segment data size=7185, suffix heap size=158696,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1257, new segment cells count=157 2024-12-06T14:43:26,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742197_1373 (size=31651) 2024-12-06T14:43:26,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742198_1374 (size=12697) 2024-12-06T14:43:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:26,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ff00b2ae4b744fb683cc3fe0df6e5eb6_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496206797/Put/seqid=0 2024-12-06T14:43:26,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742199_1375 (size=14994) 2024-12-06T14:43:26,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:26,819 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ff00b2ae4b744fb683cc3fe0df6e5eb6_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ff00b2ae4b744fb683cc3fe0df6e5eb6_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:26,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/a3c4de4f2187432ebd23bcf4f1fd5bbf, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:26,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/a3c4de4f2187432ebd23bcf4f1fd5bbf is 175, key is test_row_0/A:col10/1733496206797/Put/seqid=0 2024-12-06T14:43:26,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742200_1376 (size=39949) 2024-12-06T14:43:26,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496206830, max timestamp=1733496206844Num uniques 859; 2024-12-06T14:43:26,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:26,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496206830, max timestamp=1733496206844Num uniques 859; 2024-12-06T14:43:26,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:26,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496206830, max timestamp=1733496206844Num uniques 859; 2024-12-06T14:43:26,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:26,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:26,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496206869, max timestamp=1733496206888Num uniques 1018; 2024-12-06T14:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496206869, max timestamp=1733496206888Num uniques 1018; 2024-12-06T14:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733496206869, max timestamp=1733496206888Num uniques 1018; 2024-12-06T14:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T14:43:26,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733496206919, max timestamp=1733496206942Num uniques 1077; 2024-12-06T14:43:26,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-06T14:43:26,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733496206919, max timestamp=1733496206942Num uniques 1077; 2024-12-06T14:43:26,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733496206919, max timestamp=1733496206942Num uniques 1077; 2024-12-06T14:43:26,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-06T14:43:26,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-06T14:43:26,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496266973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496266976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:26,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733496206972, max timestamp=1733496206981Num uniques 1136; 2024-12-06T14:43:26,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733496206972, max timestamp=1733496206981Num uniques 1136; 2024-12-06T14:43:26,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-06T14:43:26,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-06T14:43:26,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733496206972, max timestamp=1733496206981Num uniques 1136; 2024-12-06T14:43:26,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-06T14:43:27,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733496207041, max timestamp=1733496207085Num uniques 1045; 2024-12-06T14:43:27,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733496207041, max timestamp=1733496207085Num uniques 1045; 2024-12-06T14:43:27,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733496207041, max timestamp=1733496207085Num uniques 1045; 2024-12-06T14:43:27,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-06T14:43:27,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-06T14:43:27,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-06T14:43:27,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1733496207135, max timestamp=1733496207176Num uniques 1054; 2024-12-06T14:43:27,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=16210, suffix heap size=137584,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=354 2024-12-06T14:43:27,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1733496207135, max timestamp=1733496207176Num uniques 1054; 2024-12-06T14:43:27,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=16210, suffix heap size=137584,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=354 2024-12-06T14:43:27,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1733496207135, max timestamp=1733496207176Num uniques 1054; 2024-12-06T14:43:27,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=16210, suffix heap size=137584,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=354 2024-12-06T14:43:27,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,216 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/1f72a205e50b4837a7f6bf1d7463c975 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/1f72a205e50b4837a7f6bf1d7463c975 2024-12-06T14:43:27,220 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/fea1e4d480af4378bb9f865f396bf9c3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fea1e4d480af4378bb9f865f396bf9c3 2024-12-06T14:43:27,223 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 1f72a205e50b4837a7f6bf1d7463c975(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:27,223 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:27,223 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=13, startTime=1733496206755; duration=0sec 2024-12-06T14:43:27,223 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:27,223 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:27,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=639, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/a3c4de4f2187432ebd23bcf4f1fd5bbf 2024-12-06T14:43:27,227 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into fea1e4d480af4378bb9f865f396bf9c3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:27,227 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:27,227 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=13, startTime=1733496206755; duration=0sec 2024-12-06T14:43:27,227 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:27,227 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:27,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/9ad82ccae5224de181d670207db13235 is 50, key is test_row_0/B:col10/1733496206797/Put/seqid=0 2024-12-06T14:43:27,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496267220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496267229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496267233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496267237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742201_1377 (size=12301) 2024-12-06T14:43:27,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496267348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496267348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496267348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496267358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496267558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496267558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496267568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496267578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/9ad82ccae5224de181d670207db13235 2024-12-06T14:43:27,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/69c255c845844e5c82b9ba22a2237876 is 50, key is test_row_0/C:col10/1733496206797/Put/seqid=0 2024-12-06T14:43:27,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742202_1378 (size=12301) 2024-12-06T14:43:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:27,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496267869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496267878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496267888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496267899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:27,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:27,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:28,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/69c255c845844e5c82b9ba22a2237876 2024-12-06T14:43:28,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/a3c4de4f2187432ebd23bcf4f1fd5bbf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf 2024-12-06T14:43:28,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf, entries=200, sequenceid=639, filesize=39.0 K 2024-12-06T14:43:28,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/9ad82ccae5224de181d670207db13235 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9ad82ccae5224de181d670207db13235 2024-12-06T14:43:28,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9ad82ccae5224de181d670207db13235, entries=150, sequenceid=639, filesize=12.0 K 2024-12-06T14:43:28,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/69c255c845844e5c82b9ba22a2237876 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/69c255c845844e5c82b9ba22a2237876 2024-12-06T14:43:28,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/69c255c845844e5c82b9ba22a2237876, entries=150, sequenceid=639, filesize=12.0 K 2024-12-06T14:43:28,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.30 KB/88368, currentSize=154.31 KB/158010 for 415d8e5bcca7a5e3eb923a4249401464 in 1312ms, sequenceid=639, compaction requested=false 2024-12-06T14:43:28,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:28,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T14:43:28,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:28,113 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=154.31 KB heapSize=426.47 KB 2024-12-06T14:43:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206caf8dc4fca2c47cb935954b831b1faea_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496207225/Put/seqid=0 2024-12-06T14:43:28,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742203_1379 (size=12454) 2024-12-06T14:43:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:28,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496268402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496268404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496268405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496268418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496268519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496268519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496268519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:28,554 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206caf8dc4fca2c47cb935954b831b1faea_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206caf8dc4fca2c47cb935954b831b1faea_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/fb07f2ae04b440c5a91987c5343b4a53, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/fb07f2ae04b440c5a91987c5343b4a53 is 175, key is test_row_0/A:col10/1733496207225/Put/seqid=0 2024-12-06T14:43:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742204_1380 (size=31255) 2024-12-06T14:43:28,564 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=763, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/fb07f2ae04b440c5a91987c5343b4a53 2024-12-06T14:43:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/bcbe442a42fc45b9ae118e7b58d41e50 is 50, key is test_row_0/B:col10/1733496207225/Put/seqid=0 2024-12-06T14:43:28,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742205_1381 (size=12301) 2024-12-06T14:43:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496268738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496268738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496268739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:28,990 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/bcbe442a42fc45b9ae118e7b58d41e50 2024-12-06T14:43:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/5ca7156704b74e59a09d4981bb42fdbf is 50, key is test_row_0/C:col10/1733496207225/Put/seqid=0 2024-12-06T14:43:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742206_1382 (size=12301) 2024-12-06T14:43:29,007 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/5ca7156704b74e59a09d4981bb42fdbf 2024-12-06T14:43:29,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/fb07f2ae04b440c5a91987c5343b4a53 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53 2024-12-06T14:43:29,018 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53, entries=150, sequenceid=763, filesize=30.5 K 2024-12-06T14:43:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/bcbe442a42fc45b9ae118e7b58d41e50 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bcbe442a42fc45b9ae118e7b58d41e50 2024-12-06T14:43:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,023 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bcbe442a42fc45b9ae118e7b58d41e50, entries=150, sequenceid=763, filesize=12.0 K 2024-12-06T14:43:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/5ca7156704b74e59a09d4981bb42fdbf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5ca7156704b74e59a09d4981bb42fdbf 2024-12-06T14:43:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,029 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5ca7156704b74e59a09d4981bb42fdbf, entries=150, sequenceid=763, filesize=12.0 K 2024-12-06T14:43:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,030 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~426.42 KB/436656, currentSize=33.54 KB/34350 for 415d8e5bcca7a5e3eb923a4249401464 in 918ms, sequenceid=763, compaction requested=true 2024-12-06T14:43:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-06T14:43:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-06T14:43:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-06T14:43:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3790 sec 2024-12-06T14:43:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.3850 sec 2024-12-06T14:43:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dddf6ca10355499d877c8178eafb4037_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742207_1383 (size=14994) 2024-12-06T14:43:29,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496209118, max timestamp=1733496209132Num uniques 859; 2024-12-06T14:43:29,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496209118, max timestamp=1733496209132Num uniques 859; 2024-12-06T14:43:29,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:29,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:29,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496209118, max timestamp=1733496209132Num uniques 859; 2024-12-06T14:43:29,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:29,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496269181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496269189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496269190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496269298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496269298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496269308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496269438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,440 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 415d8e5bcca7a5e3eb923a4249401464 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:29,488 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dddf6ca10355499d877c8178eafb4037_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dddf6ca10355499d877c8178eafb4037_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:29,489 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3438f4dcc82d4a66bddc2774ed7cdf25, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:29,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3438f4dcc82d4a66bddc2774ed7cdf25 is 175, key is test_row_0/A:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742208_1384 (size=39949) 2024-12-06T14:43:29,495 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=773, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3438f4dcc82d4a66bddc2774ed7cdf25 2024-12-06T14:43:29,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/08156408267e49d38afb2645df87bab2 is 50, key is test_row_0/B:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:29,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742209_1385 (size=12301) 2024-12-06T14:43:29,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496269518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496269518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496269518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T14:43:29,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:29,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-06T14:43:29,821 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T14:43:29,822 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:29,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:29,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496269828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496269838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496269839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:29,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/08156408267e49d38afb2645df87bab2 2024-12-06T14:43:29,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/11157146801d4e539bdcc8411e632c18 is 50, key is test_row_0/C:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:29,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742210_1386 (size=12301) 2024-12-06T14:43:29,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/11157146801d4e539bdcc8411e632c18 2024-12-06T14:43:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T14:43:29,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/3438f4dcc82d4a66bddc2774ed7cdf25 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25 2024-12-06T14:43:29,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25, entries=200, sequenceid=773, filesize=39.0 K 2024-12-06T14:43:29,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/08156408267e49d38afb2645df87bab2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/08156408267e49d38afb2645df87bab2 2024-12-06T14:43:29,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/08156408267e49d38afb2645df87bab2, entries=150, sequenceid=773, filesize=12.0 K 2024-12-06T14:43:29,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/11157146801d4e539bdcc8411e632c18 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/11157146801d4e539bdcc8411e632c18 2024-12-06T14:43:29,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/11157146801d4e539bdcc8411e632c18, entries=150, sequenceid=773, filesize=12.0 K 2024-12-06T14:43:29,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 415d8e5bcca7a5e3eb923a4249401464 in 881ms, sequenceid=773, compaction requested=true 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:29,947 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:29,947 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142804 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:29,949 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:29,949 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:29,949 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9074a01d12314a43aaaf1e3cf528e8c4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9ad82ccae5224de181d670207db13235, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bcbe442a42fc45b9ae118e7b58d41e50, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/08156408267e49d38afb2645df87bab2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=48.4 K 2024-12-06T14:43:29,949 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/1f72a205e50b4837a7f6bf1d7463c975, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=139.5 K 2024-12-06T14:43:29,949 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/1f72a205e50b4837a7f6bf1d7463c975, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25] 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9074a01d12314a43aaaf1e3cf528e8c4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733496204744 2024-12-06T14:43:29,949 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f72a205e50b4837a7f6bf1d7463c975, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733496204744 2024-12-06T14:43:29,950 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad82ccae5224de181d670207db13235, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733496206660 2024-12-06T14:43:29,950 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c4de4f2187432ebd23bcf4f1fd5bbf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733496206660 2024-12-06T14:43:29,950 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting bcbe442a42fc45b9ae118e7b58d41e50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733496207225 2024-12-06T14:43:29,950 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb07f2ae04b440c5a91987c5343b4a53, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733496207225 2024-12-06T14:43:29,950 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 08156408267e49d38afb2645df87bab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1733496208402 2024-12-06T14:43:29,950 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3438f4dcc82d4a66bddc2774ed7cdf25, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1733496208402 2024-12-06T14:43:29,956 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:29,958 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:29,958 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/a0ae8350ca80430f897f9f66c290a60e is 50, key is test_row_0/B:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:29,961 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412065b5310a5da1c4783a062c8921ace6240_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:29,963 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412065b5310a5da1c4783a062c8921ace6240_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:29,963 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065b5310a5da1c4783a062c8921ace6240_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:29,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T14:43:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:29,974 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-06T14:43:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:29,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742211_1387 (size=12983) 2024-12-06T14:43:29,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742212_1388 (size=4469) 2024-12-06T14:43:29,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120640d3891f15b14b74a44a8a42f4f2398b_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496209162/Put/seqid=0 2024-12-06T14:43:29,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742213_1389 (size=12454) 2024-12-06T14:43:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T14:43:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:30,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:30,379 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:30,379 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/4fd58fc5047b4cf09de8122db3ceba5c is 175, key is test_row_0/A:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:30,381 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/a0ae8350ca80430f897f9f66c290a60e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a0ae8350ca80430f897f9f66c290a60e 2024-12-06T14:43:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496270371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742214_1390 (size=31937) 2024-12-06T14:43:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:30,390 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into a0ae8350ca80430f897f9f66c290a60e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:30,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:30,390 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=12, startTime=1733496209947; duration=0sec 2024-12-06T14:43:30,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:30,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:30,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:43:30,391 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:43:30,391 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:30,391 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:30,391 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fea1e4d480af4378bb9f865f396bf9c3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/69c255c845844e5c82b9ba22a2237876, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5ca7156704b74e59a09d4981bb42fdbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/11157146801d4e539bdcc8411e632c18] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=48.4 K 2024-12-06T14:43:30,392 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting fea1e4d480af4378bb9f865f396bf9c3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733496204744 2024-12-06T14:43:30,393 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c255c845844e5c82b9ba22a2237876, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733496206660 2024-12-06T14:43:30,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496270386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,401 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120640d3891f15b14b74a44a8a42f4f2398b_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120640d3891f15b14b74a44a8a42f4f2398b_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:30,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496270386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,405 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca7156704b74e59a09d4981bb42fdbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733496207225 2024-12-06T14:43:30,406 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 11157146801d4e539bdcc8411e632c18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1733496208402 2024-12-06T14:43:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/84725dd7d8a44008844b3b58dfd16eba, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/84725dd7d8a44008844b3b58dfd16eba is 175, key is test_row_0/A:col10/1733496209162/Put/seqid=0 2024-12-06T14:43:30,411 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/4fd58fc5047b4cf09de8122db3ceba5c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/4fd58fc5047b4cf09de8122db3ceba5c 2024-12-06T14:43:30,419 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 4fd58fc5047b4cf09de8122db3ceba5c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:30,419 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:30,419 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=12, startTime=1733496209947; duration=0sec 2024-12-06T14:43:30,419 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:30,419 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:30,427 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:30,428 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/58abe2c3c2e1415ea9b45fa871834b18 is 50, key is test_row_0/C:col10/1733496209063/Put/seqid=0 2024-12-06T14:43:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T14:43:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742215_1391 (size=31255) 2024-12-06T14:43:30,458 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=810, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/84725dd7d8a44008844b3b58dfd16eba 2024-12-06T14:43:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/042d2645f1714fcfab2bc11f82106abb is 50, key is test_row_0/B:col10/1733496209162/Put/seqid=0 2024-12-06T14:43:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496270489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496270509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496270509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742216_1392 (size=12983) 2024-12-06T14:43:30,550 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/58abe2c3c2e1415ea9b45fa871834b18 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/58abe2c3c2e1415ea9b45fa871834b18 2024-12-06T14:43:30,556 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into 58abe2c3c2e1415ea9b45fa871834b18(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:30,556 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:30,556 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=12, startTime=1733496209947; duration=0sec 2024-12-06T14:43:30,557 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:30,557 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:30,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742217_1393 (size=12301) 2024-12-06T14:43:30,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496270709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496270728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496270728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T14:43:30,990 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=810 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/042d2645f1714fcfab2bc11f82106abb 2024-12-06T14:43:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/cd85530b014543e6b5fc27bc66675dc0 is 50, key is test_row_0/C:col10/1733496209162/Put/seqid=0 2024-12-06T14:43:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496271028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742218_1394 (size=12301) 2024-12-06T14:43:31,043 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=810 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/cd85530b014543e6b5fc27bc66675dc0 2024-12-06T14:43:31,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496271038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/84725dd7d8a44008844b3b58dfd16eba as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba 2024-12-06T14:43:31,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496271049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,057 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba, entries=150, sequenceid=810, filesize=30.5 K 2024-12-06T14:43:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/042d2645f1714fcfab2bc11f82106abb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/042d2645f1714fcfab2bc11f82106abb 2024-12-06T14:43:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,062 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/042d2645f1714fcfab2bc11f82106abb, entries=150, sequenceid=810, filesize=12.0 K 2024-12-06T14:43:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/cd85530b014543e6b5fc27bc66675dc0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/cd85530b014543e6b5fc27bc66675dc0 2024-12-06T14:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/cd85530b014543e6b5fc27bc66675dc0, entries=150, sequenceid=810, filesize=12.0 K 2024-12-06T14:43:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 415d8e5bcca7a5e3eb923a4249401464 in 1092ms, sequenceid=810, compaction requested=false 2024-12-06T14:43:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-06T14:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-06T14:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-06T14:43:31,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-12-06T14:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2520 sec 2024-12-06T14:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:31,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:43:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fba626e03ed44e6483e550cf1d97d6cc_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742219_1395 (size=12454) 2024-12-06T14:43:31,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496271583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496271591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496271592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496271598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496271708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496271708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496271708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496271718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496271928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496271928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496271929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:31,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fba626e03ed44e6483e550cf1d97d6cc_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fba626e03ed44e6483e550cf1d97d6cc_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:31,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6b52db73e07946068d2374219ceba33f, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:31,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6b52db73e07946068d2374219ceba33f is 175, key is test_row_0/A:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496271938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:31,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742220_1396 (size=31255) 2024-12-06T14:43:31,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=825, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6b52db73e07946068d2374219ceba33f 2024-12-06T14:43:31,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/db35b5c7e81845a38aae2642c0d7cb2b is 50, key is test_row_0/B:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:31,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T14:43:31,968 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:31,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-06T14:43:31,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T14:43:31,971 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:31,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:31,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742221_1397 (size=12301) 2024-12-06T14:43:31,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=825 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/db35b5c7e81845a38aae2642c0d7cb2b 2024-12-06T14:43:31,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/49cefcc392af4b1493070032379b7054 is 50, key is test_row_0/C:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:32,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742222_1398 (size=12301) 2024-12-06T14:43:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T14:43:32,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T14:43:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496272238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496272238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496272238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496272249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T14:43:32,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:32,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T14:43:32,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=825 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/49cefcc392af4b1493070032379b7054 2024-12-06T14:43:32,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/6b52db73e07946068d2374219ceba33f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f 2024-12-06T14:43:32,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f, entries=150, sequenceid=825, filesize=30.5 K 2024-12-06T14:43:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/db35b5c7e81845a38aae2642c0d7cb2b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/db35b5c7e81845a38aae2642c0d7cb2b 2024-12-06T14:43:32,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/db35b5c7e81845a38aae2642c0d7cb2b, entries=150, sequenceid=825, filesize=12.0 K 2024-12-06T14:43:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/49cefcc392af4b1493070032379b7054 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/49cefcc392af4b1493070032379b7054 2024-12-06T14:43:32,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T14:43:32,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:32,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:32,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/49cefcc392af4b1493070032379b7054, entries=150, sequenceid=825, filesize=12.0 K 2024-12-06T14:43:32,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 415d8e5bcca7a5e3eb923a4249401464 in 954ms, sequenceid=825, compaction requested=true 2024-12-06T14:43:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:32,435 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:32,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:32,437 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:32,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:32,437 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:32,437 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:32,437 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,437 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:32,438 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:32,438 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/4fd58fc5047b4cf09de8122db3ceba5c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=92.2 K 2024-12-06T14:43:32,438 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,438 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,438 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a0ae8350ca80430f897f9f66c290a60e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/042d2645f1714fcfab2bc11f82106abb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/db35b5c7e81845a38aae2642c0d7cb2b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=36.7 K 2024-12-06T14:43:32,438 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/4fd58fc5047b4cf09de8122db3ceba5c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f] 2024-12-06T14:43:32,438 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fd58fc5047b4cf09de8122db3ceba5c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1733496208402 2024-12-06T14:43:32,438 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ae8350ca80430f897f9f66c290a60e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1733496208402 2024-12-06T14:43:32,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:32,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:32,439 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 042d2645f1714fcfab2bc11f82106abb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=810, earliestPutTs=1733496209162 2024-12-06T14:43:32,439 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84725dd7d8a44008844b3b58dfd16eba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=810, earliestPutTs=1733496209162 2024-12-06T14:43:32,439 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b52db73e07946068d2374219ceba33f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1733496210382 2024-12-06T14:43:32,439 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting db35b5c7e81845a38aae2642c0d7cb2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1733496210382 2024-12-06T14:43:32,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:32,447 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:32,451 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:32,451 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/b051113bb62248c7a733e99207247a51 is 50, key is test_row_0/B:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:32,453 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206a3ca7181293f4dcf8d2debee449c95b0_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:32,455 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206a3ca7181293f4dcf8d2debee449c95b0_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:32,455 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a3ca7181293f4dcf8d2debee449c95b0_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:32,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742223_1399 (size=13085) 2024-12-06T14:43:32,487 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/b051113bb62248c7a733e99207247a51 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b051113bb62248c7a733e99207247a51 2024-12-06T14:43:32,492 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into b051113bb62248c7a733e99207247a51(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:32,492 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:32,492 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=13, startTime=1733496212437; duration=0sec 2024-12-06T14:43:32,492 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:32,492 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:32,492 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:32,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742224_1400 (size=4469) 2024-12-06T14:43:32,495 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#333 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:32,495 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:32,496 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:32,496 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/9e5efb71735a40a8a4ffe22f4710d921 is 175, key is test_row_0/A:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:32,496 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,496 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/58abe2c3c2e1415ea9b45fa871834b18, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/cd85530b014543e6b5fc27bc66675dc0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/49cefcc392af4b1493070032379b7054] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=36.7 K 2024-12-06T14:43:32,497 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 58abe2c3c2e1415ea9b45fa871834b18, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1733496208402 2024-12-06T14:43:32,498 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cd85530b014543e6b5fc27bc66675dc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=810, earliestPutTs=1733496209162 2024-12-06T14:43:32,498 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 49cefcc392af4b1493070032379b7054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1733496210382 2024-12-06T14:43:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742225_1401 (size=32039) 2024-12-06T14:43:32,522 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/9e5efb71735a40a8a4ffe22f4710d921 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9e5efb71735a40a8a4ffe22f4710d921 2024-12-06T14:43:32,522 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:32,523 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/e4b3a1222d5f415e889756592ba8d882 is 50, key is test_row_0/C:col10/1733496210382/Put/seqid=0 2024-12-06T14:43:32,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742226_1402 (size=13085) 2024-12-06T14:43:32,538 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 9e5efb71735a40a8a4ffe22f4710d921(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:32,538 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:32,538 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=13, startTime=1733496212435; duration=0sec 2024-12-06T14:43:32,539 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:32,539 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:32,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T14:43:32,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:32,583 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T14:43:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:32,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d096dc2af158431ea53ce66763c6a794_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496211582/Put/seqid=0 2024-12-06T14:43:32,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742227_1403 (size=12454) 2024-12-06T14:43:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T14:43:32,601 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d096dc2af158431ea53ce66763c6a794_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d096dc2af158431ea53ce66763c6a794_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:32,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2e62c1239f8a45d0ab6013da2ef3a7aa, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:32,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2e62c1239f8a45d0ab6013da2ef3a7aa is 175, key is test_row_0/A:col10/1733496211582/Put/seqid=0 2024-12-06T14:43:32,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742228_1404 (size=31255) 2024-12-06T14:43:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:32,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:32,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496272769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496272770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496272774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496272775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496272878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496272878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496272878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496272888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:32,942 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/e4b3a1222d5f415e889756592ba8d882 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e4b3a1222d5f415e889756592ba8d882 2024-12-06T14:43:32,946 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into e4b3a1222d5f415e889756592ba8d882(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:32,946 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:32,946 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=13, startTime=1733496212439; duration=0sec 2024-12-06T14:43:32,946 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:32,946 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:33,014 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=847, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2e62c1239f8a45d0ab6013da2ef3a7aa 2024-12-06T14:43:33,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/1f66eba9dbde47ffad8ebe0938020382 is 50, key is test_row_0/B:col10/1733496211582/Put/seqid=0 2024-12-06T14:43:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742229_1405 (size=12301) 2024-12-06T14:43:33,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496273088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496273088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496273088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496273099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T14:43:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496273398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496273398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496273399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496273410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:33,439 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=847 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/1f66eba9dbde47ffad8ebe0938020382 2024-12-06T14:43:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/33babd009f1a43798716e6a3d0ea7fd0 is 50, key is test_row_0/C:col10/1733496211582/Put/seqid=0 2024-12-06T14:43:33,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742230_1406 (size=12301) 2024-12-06T14:43:33,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=847 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/33babd009f1a43798716e6a3d0ea7fd0 2024-12-06T14:43:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2e62c1239f8a45d0ab6013da2ef3a7aa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa 2024-12-06T14:43:33,470 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa, entries=150, sequenceid=847, filesize=30.5 K 2024-12-06T14:43:33,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/1f66eba9dbde47ffad8ebe0938020382 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/1f66eba9dbde47ffad8ebe0938020382 2024-12-06T14:43:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,474 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/1f66eba9dbde47ffad8ebe0938020382, entries=150, sequenceid=847, filesize=12.0 K 2024-12-06T14:43:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/33babd009f1a43798716e6a3d0ea7fd0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/33babd009f1a43798716e6a3d0ea7fd0 2024-12-06T14:43:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,493 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/33babd009f1a43798716e6a3d0ea7fd0, entries=150, sequenceid=847, filesize=12.0 K 2024-12-06T14:43:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,494 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 415d8e5bcca7a5e3eb923a4249401464 in 911ms, sequenceid=847, compaction requested=false 2024-12-06T14:43:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-06T14:43:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-06T14:43:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-12-06T14:43:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5300 sec 2024-12-06T14:43:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T14:43:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:33,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a35c32c70a7648eca3e713e795907a7d_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496273990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496273991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496273996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496273997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742231_1407 (size=12454) 2024-12-06T14:43:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496274109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496274109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T14:43:34,120 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:34,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-06T14:43:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:34,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496274128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496274128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,133 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:34,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46624 deadline: 1733496274189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,196 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 415d8e5bcca7a5e3eb923a4249401464 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18351 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:34,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496274318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496274318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496274339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496274339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:34,412 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a35c32c70a7648eca3e713e795907a7d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a35c32c70a7648eca3e713e795907a7d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:34,413 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2693d9d2a9da415c9dce43c79c21377b, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:34,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2693d9d2a9da415c9dce43c79c21377b is 175, key is test_row_0/A:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:34,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742232_1408 (size=31255) 2024-12-06T14:43:34,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=861, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2693d9d2a9da415c9dce43c79c21377b 2024-12-06T14:43:34,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/8c02b9c313e94c439f198d6e5a7d25ae is 50, key is test_row_0/B:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:34,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742233_1409 (size=12301) 2024-12-06T14:43:34,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:34,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496274638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496274638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496274658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496274658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:34,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:34,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=861 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/8c02b9c313e94c439f198d6e5a7d25ae 2024-12-06T14:43:34,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/9b9f35889c5d47cc918a37429d84d352 is 50, key is test_row_0/C:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:34,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742234_1410 (size=12301) 2024-12-06T14:43:34,895 INFO [Thread-1421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1421. 2024-12-06T14:43:34,895 DEBUG [Thread-1421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:34,895 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,895 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:34,900 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-12-06T14:43:34,900 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:34,901 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,901 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:34,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:34,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:34,903 INFO [Thread-1429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1429. 2024-12-06T14:43:34,903 DEBUG [Thread-1429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:34,903 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,903 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:34,906 INFO [Thread-1427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1427. 2024-12-06T14:43:34,906 DEBUG [Thread-1427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:34,906 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,906 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:34,912 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-12-06T14:43:34,912 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:34,912 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,913 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:34,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:35,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:35,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1733496275148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:35,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1733496275158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46634 deadline: 1733496275178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46588 deadline: 1733496275178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:43:35,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:35,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=861 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/9b9f35889c5d47cc918a37429d84d352 2024-12-06T14:43:35,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/2693d9d2a9da415c9dce43c79c21377b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b 2024-12-06T14:43:35,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b, entries=150, sequenceid=861, filesize=30.5 K 2024-12-06T14:43:35,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/8c02b9c313e94c439f198d6e5a7d25ae as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8c02b9c313e94c439f198d6e5a7d25ae 2024-12-06T14:43:35,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8c02b9c313e94c439f198d6e5a7d25ae, entries=150, sequenceid=861, filesize=12.0 K 2024-12-06T14:43:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:35,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/9b9f35889c5d47cc918a37429d84d352 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/9b9f35889c5d47cc918a37429d84d352 2024-12-06T14:43:35,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/9b9f35889c5d47cc918a37429d84d352, entries=150, sequenceid=861, filesize=12.0 K 2024-12-06T14:43:35,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 415d8e5bcca7a5e3eb923a4249401464 in 1335ms, sequenceid=861, compaction requested=true 2024-12-06T14:43:35,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:35,274 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:35,274 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 415d8e5bcca7a5e3eb923a4249401464:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:35,274 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/A is initiating minor compaction (all files) 2024-12-06T14:43:35,275 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/A in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,275 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9e5efb71735a40a8a4ffe22f4710d921, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=92.3 K 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:35,275 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/B is initiating minor compaction (all files) 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9e5efb71735a40a8a4ffe22f4710d921, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b] 2024-12-06T14:43:35,275 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/B in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,275 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b051113bb62248c7a733e99207247a51, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/1f66eba9dbde47ffad8ebe0938020382, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8c02b9c313e94c439f198d6e5a7d25ae] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=36.8 K 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5efb71735a40a8a4ffe22f4710d921, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1733496210382 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b051113bb62248c7a733e99207247a51, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1733496210382 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e62c1239f8a45d0ab6013da2ef3a7aa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=847, earliestPutTs=1733496211582 2024-12-06T14:43:35,275 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f66eba9dbde47ffad8ebe0938020382, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=847, earliestPutTs=1733496211582 2024-12-06T14:43:35,276 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2693d9d2a9da415c9dce43c79c21377b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=861, earliestPutTs=1733496212773 2024-12-06T14:43:35,276 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c02b9c313e94c439f198d6e5a7d25ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=861, earliestPutTs=1733496212773 2024-12-06T14:43:35,281 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:35,282 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:35,282 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/79daa967fdcd4f8f81642c2cd8e9efaf is 50, key is test_row_0/B:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:35,284 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412061d98b1c204aa42f88033f1a4f31fb2f6_415d8e5bcca7a5e3eb923a4249401464 store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:35,288 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412061d98b1c204aa42f88033f1a4f31fb2f6_415d8e5bcca7a5e3eb923a4249401464, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:35,289 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061d98b1c204aa42f88033f1a4f31fb2f6_415d8e5bcca7a5e3eb923a4249401464 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:35,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742235_1411 (size=13187) 2024-12-06T14:43:35,298 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/79daa967fdcd4f8f81642c2cd8e9efaf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/79daa967fdcd4f8f81642c2cd8e9efaf 2024-12-06T14:43:35,302 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/B of 415d8e5bcca7a5e3eb923a4249401464 into 79daa967fdcd4f8f81642c2cd8e9efaf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:35,302 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:35,302 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/B, priority=13, startTime=1733496215274; duration=0sec 2024-12-06T14:43:35,302 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:35,302 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:B 2024-12-06T14:43:35,302 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:35,303 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:35,303 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 415d8e5bcca7a5e3eb923a4249401464/C is initiating minor compaction (all files) 2024-12-06T14:43:35,303 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 415d8e5bcca7a5e3eb923a4249401464/C in TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,303 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e4b3a1222d5f415e889756592ba8d882, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/33babd009f1a43798716e6a3d0ea7fd0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/9b9f35889c5d47cc918a37429d84d352] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp, totalSize=36.8 K 2024-12-06T14:43:35,303 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b3a1222d5f415e889756592ba8d882, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1733496210382 2024-12-06T14:43:35,303 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 33babd009f1a43798716e6a3d0ea7fd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=847, earliestPutTs=1733496211582 2024-12-06T14:43:35,303 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9f35889c5d47cc918a37429d84d352, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=861, earliestPutTs=1733496212773 2024-12-06T14:43:35,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742236_1412 (size=4469) 2024-12-06T14:43:35,305 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#A#compaction#343 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:35,306 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/7c4f296fee4245be8330b3c1d18eb936 is 175, key is test_row_0/A:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742237_1413 (size=32141) 2024-12-06T14:43:35,318 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 415d8e5bcca7a5e3eb923a4249401464#C#compaction#344 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:35,318 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/0a3c30f418d94ef5b06b9fc49b382af2 is 50, key is test_row_0/C:col10/1733496212773/Put/seqid=0 2024-12-06T14:43:35,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742238_1414 (size=13187) 2024-12-06T14:43:35,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T14:43:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:35,359 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:43:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:35,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064fd05021057c4608b594942e9f06df61_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496213996/Put/seqid=0 2024-12-06T14:43:35,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742239_1415 (size=12454) 2024-12-06T14:43:35,720 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/7c4f296fee4245be8330b3c1d18eb936 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7c4f296fee4245be8330b3c1d18eb936 2024-12-06T14:43:35,725 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/A of 415d8e5bcca7a5e3eb923a4249401464 into 7c4f296fee4245be8330b3c1d18eb936(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:35,725 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:35,725 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/A, priority=13, startTime=1733496215273; duration=0sec 2024-12-06T14:43:35,725 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:35,725 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:A 2024-12-06T14:43:35,725 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/0a3c30f418d94ef5b06b9fc49b382af2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/0a3c30f418d94ef5b06b9fc49b382af2 2024-12-06T14:43:35,728 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 415d8e5bcca7a5e3eb923a4249401464/C of 415d8e5bcca7a5e3eb923a4249401464 into 0a3c30f418d94ef5b06b9fc49b382af2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:35,728 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:35,728 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., storeName=415d8e5bcca7a5e3eb923a4249401464/C, priority=13, startTime=1733496215274; duration=0sec 2024-12-06T14:43:35,728 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:35,728 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 415d8e5bcca7a5e3eb923a4249401464:C 2024-12-06T14:43:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:35,775 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064fd05021057c4608b594942e9f06df61_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064fd05021057c4608b594942e9f06df61_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:35,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/35e93b8c52ea46629465e8f1ce5b36bc, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/35e93b8c52ea46629465e8f1ce5b36bc is 175, key is test_row_0/A:col10/1733496213996/Put/seqid=0 2024-12-06T14:43:35,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742240_1416 (size=31255) 2024-12-06T14:43:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. as already flushing 2024-12-06T14:43:36,160 INFO [Thread-1412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1412. 2024-12-06T14:43:36,160 DEBUG [Thread-1412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:36,160 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,160 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:36,170 INFO [Thread-1416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1416. 2024-12-06T14:43:36,170 DEBUG [Thread-1416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:36,170 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,170 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:36,183 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=881, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/35e93b8c52ea46629465e8f1ce5b36bc 2024-12-06T14:43:36,190 INFO [Thread-1410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1410. 2024-12-06T14:43:36,190 DEBUG [Thread-1410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:36,190 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,190 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:36,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/f252e3f1de1e417bb1089c218be84f30 is 50, key is test_row_0/B:col10/1733496213996/Put/seqid=0 2024-12-06T14:43:36,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742241_1417 (size=12301) 2024-12-06T14:43:36,199 INFO [Thread-1418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1418. 2024-12-06T14:43:36,199 DEBUG [Thread-1418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:36,199 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,199 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:36,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:36,595 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=881 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/f252e3f1de1e417bb1089c218be84f30 2024-12-06T14:43:36,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/01997f54ed5547e68c2c5999fd286cfe is 50, key is test_row_0/C:col10/1733496213996/Put/seqid=0 2024-12-06T14:43:36,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742242_1418 (size=12301) 2024-12-06T14:43:37,005 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=881 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/01997f54ed5547e68c2c5999fd286cfe 2024-12-06T14:43:37,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/35e93b8c52ea46629465e8f1ce5b36bc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/35e93b8c52ea46629465e8f1ce5b36bc 2024-12-06T14:43:37,011 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/35e93b8c52ea46629465e8f1ce5b36bc, entries=150, sequenceid=881, filesize=30.5 K 2024-12-06T14:43:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/f252e3f1de1e417bb1089c218be84f30 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f252e3f1de1e417bb1089c218be84f30 2024-12-06T14:43:37,015 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f252e3f1de1e417bb1089c218be84f30, entries=150, sequenceid=881, filesize=12.0 K 2024-12-06T14:43:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/01997f54ed5547e68c2c5999fd286cfe as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/01997f54ed5547e68c2c5999fd286cfe 2024-12-06T14:43:37,019 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/01997f54ed5547e68c2c5999fd286cfe, entries=150, sequenceid=881, filesize=12.0 K 2024-12-06T14:43:37,020 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for 415d8e5bcca7a5e3eb923a4249401464 in 1661ms, sequenceid=881, compaction requested=false 2024-12-06T14:43:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 415d8e5bcca7a5e3eb923a4249401464: 2024-12-06T14:43:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-06T14:43:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-06T14:43:37,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-06T14:43:37,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8880 sec 2024-12-06T14:43:37,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.9020 sec 2024-12-06T14:43:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T14:43:38,299 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:38,547 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T14:43:43,322 WARN [regionserver/bb7ea3e46d9c:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-12-06T14:43:43,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9c94431a11284a8ca7279c016c10c116, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7ea1f633a1dd4e719ddf34a453ec00bb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/43e4690a04ca45a581604ed81446cfeb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/c7e15711e9a346eb800b33847017a8fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/1f72a205e50b4837a7f6bf1d7463c975, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/4fd58fc5047b4cf09de8122db3ceba5c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9e5efb71735a40a8a4ffe22f4710d921, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b] to archive 2024-12-06T14:43:43,332 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:43:43,333 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3a11efba1a3c4a4eb408df8f9c5b7004 2024-12-06T14:43:43,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ba2346b3df994a4eae5ae6139aa2d953 2024-12-06T14:43:43,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/03f85eead85a436aaca888dc4193c5bf 2024-12-06T14:43:43,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9c94431a11284a8ca7279c016c10c116 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9c94431a11284a8ca7279c016c10c116 2024-12-06T14:43:43,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/47696a3d8c7e47bc9c93163d016ee70c 2024-12-06T14:43:43,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/774c9db3c36a4698b18033237a87beba 2024-12-06T14:43:43,338 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7ea1f633a1dd4e719ddf34a453ec00bb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7ea1f633a1dd4e719ddf34a453ec00bb 2024-12-06T14:43:43,339 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6d5ee9b2b4b44423bf5733b68d0deb04 2024-12-06T14:43:43,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/408da551bef84f4f84a3a549bf75e052 2024-12-06T14:43:43,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/43e4690a04ca45a581604ed81446cfeb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/43e4690a04ca45a581604ed81446cfeb 2024-12-06T14:43:43,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/ad31f2cd8c09484981b1a02edb9369aa 2024-12-06T14:43:43,342 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/329a88f240894966ab1e4ea314390510 2024-12-06T14:43:43,343 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/f89ffe1382b14870866b7aeda2c17fe4 2024-12-06T14:43:43,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/c7e15711e9a346eb800b33847017a8fa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/c7e15711e9a346eb800b33847017a8fa 2024-12-06T14:43:43,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/8421951a7e524ff19c63229368693d3e 2024-12-06T14:43:43,345 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/30181bf020d8430d83c49ae45b2fa1d5 2024-12-06T14:43:43,346 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/1f72a205e50b4837a7f6bf1d7463c975 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/1f72a205e50b4837a7f6bf1d7463c975 2024-12-06T14:43:43,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/a3c4de4f2187432ebd23bcf4f1fd5bbf 2024-12-06T14:43:43,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/fb07f2ae04b440c5a91987c5343b4a53 2024-12-06T14:43:43,349 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/3438f4dcc82d4a66bddc2774ed7cdf25 2024-12-06T14:43:43,350 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/4fd58fc5047b4cf09de8122db3ceba5c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/4fd58fc5047b4cf09de8122db3ceba5c 2024-12-06T14:43:43,351 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/84725dd7d8a44008844b3b58dfd16eba 2024-12-06T14:43:43,352 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9e5efb71735a40a8a4ffe22f4710d921 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/9e5efb71735a40a8a4ffe22f4710d921 2024-12-06T14:43:43,353 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/6b52db73e07946068d2374219ceba33f 2024-12-06T14:43:43,354 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2e62c1239f8a45d0ab6013da2ef3a7aa 2024-12-06T14:43:43,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/2693d9d2a9da415c9dce43c79c21377b 2024-12-06T14:43:43,359 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/2cf9d31c3d934e9cac724e1fb9e5fed4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f560eb546ae5419580509e478791bfa3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/93ce57d6abb44c39a115743300bf40a9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a78b651ea41145539fee4f0419ff3bd5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8f4a25c939a241b2bf5f133fee38824c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/cd21a6707d1d4b35b4bc673b26d15c7e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/540ac2bad3914baaae619dc3eb584751, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dab80056cf294d388029bb2dab7c14fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6bba23e7a8314d6992eaeb85a5f7dab2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dfb7b0ea8467466dba992240cfc48cb8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dce22544f47e44869b1eb22952d6494d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/3e25cdb8553949dabd9784194c03a74b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/72f1d41ee0b64139b3979e6a5c2e4460, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bef6f0b5c7c04f559d654769d0dfcc29, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/40c0fb9119574267835688cfba642146, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9074a01d12314a43aaaf1e3cf528e8c4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6f2957719c264706b90b5c295b251811, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9ad82ccae5224de181d670207db13235, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bcbe442a42fc45b9ae118e7b58d41e50, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a0ae8350ca80430f897f9f66c290a60e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/08156408267e49d38afb2645df87bab2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/042d2645f1714fcfab2bc11f82106abb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b051113bb62248c7a733e99207247a51, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/db35b5c7e81845a38aae2642c0d7cb2b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/1f66eba9dbde47ffad8ebe0938020382, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8c02b9c313e94c439f198d6e5a7d25ae] to archive 2024-12-06T14:43:43,360 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:43:43,361 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/2cf9d31c3d934e9cac724e1fb9e5fed4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/2cf9d31c3d934e9cac724e1fb9e5fed4 2024-12-06T14:43:43,363 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f560eb546ae5419580509e478791bfa3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f560eb546ae5419580509e478791bfa3 2024-12-06T14:43:43,364 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/93ce57d6abb44c39a115743300bf40a9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/93ce57d6abb44c39a115743300bf40a9 2024-12-06T14:43:43,365 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a78b651ea41145539fee4f0419ff3bd5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a78b651ea41145539fee4f0419ff3bd5 2024-12-06T14:43:43,366 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8f4a25c939a241b2bf5f133fee38824c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8f4a25c939a241b2bf5f133fee38824c 2024-12-06T14:43:43,367 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/cd21a6707d1d4b35b4bc673b26d15c7e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/cd21a6707d1d4b35b4bc673b26d15c7e 2024-12-06T14:43:43,368 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/540ac2bad3914baaae619dc3eb584751 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/540ac2bad3914baaae619dc3eb584751 2024-12-06T14:43:43,369 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dab80056cf294d388029bb2dab7c14fa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dab80056cf294d388029bb2dab7c14fa 2024-12-06T14:43:43,369 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6bba23e7a8314d6992eaeb85a5f7dab2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6bba23e7a8314d6992eaeb85a5f7dab2 2024-12-06T14:43:43,370 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dfb7b0ea8467466dba992240cfc48cb8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dfb7b0ea8467466dba992240cfc48cb8 2024-12-06T14:43:43,371 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dce22544f47e44869b1eb22952d6494d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/dce22544f47e44869b1eb22952d6494d 2024-12-06T14:43:43,372 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/3e25cdb8553949dabd9784194c03a74b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/3e25cdb8553949dabd9784194c03a74b 2024-12-06T14:43:43,373 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/72f1d41ee0b64139b3979e6a5c2e4460 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/72f1d41ee0b64139b3979e6a5c2e4460 2024-12-06T14:43:43,374 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bef6f0b5c7c04f559d654769d0dfcc29 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bef6f0b5c7c04f559d654769d0dfcc29 2024-12-06T14:43:43,374 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/40c0fb9119574267835688cfba642146 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/40c0fb9119574267835688cfba642146 2024-12-06T14:43:43,375 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9074a01d12314a43aaaf1e3cf528e8c4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9074a01d12314a43aaaf1e3cf528e8c4 2024-12-06T14:43:43,376 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6f2957719c264706b90b5c295b251811 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/6f2957719c264706b90b5c295b251811 2024-12-06T14:43:43,377 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9ad82ccae5224de181d670207db13235 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/9ad82ccae5224de181d670207db13235 2024-12-06T14:43:43,378 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bcbe442a42fc45b9ae118e7b58d41e50 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/bcbe442a42fc45b9ae118e7b58d41e50 2024-12-06T14:43:43,379 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a0ae8350ca80430f897f9f66c290a60e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/a0ae8350ca80430f897f9f66c290a60e 2024-12-06T14:43:43,381 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/08156408267e49d38afb2645df87bab2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/08156408267e49d38afb2645df87bab2 2024-12-06T14:43:43,382 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/042d2645f1714fcfab2bc11f82106abb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/042d2645f1714fcfab2bc11f82106abb 2024-12-06T14:43:43,383 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b051113bb62248c7a733e99207247a51 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b051113bb62248c7a733e99207247a51 2024-12-06T14:43:43,384 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/db35b5c7e81845a38aae2642c0d7cb2b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/db35b5c7e81845a38aae2642c0d7cb2b 2024-12-06T14:43:43,385 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/1f66eba9dbde47ffad8ebe0938020382 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/1f66eba9dbde47ffad8ebe0938020382 2024-12-06T14:43:43,386 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8c02b9c313e94c439f198d6e5a7d25ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/8c02b9c313e94c439f198d6e5a7d25ae 2024-12-06T14:43:43,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/ea2e46c1750547e385135b47b432a618, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73182c8e24a5479da06a6efd6f745b25, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e1e006d7a3114e949114ccaa830b793c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/a33e6a24022345e9a5f7f2cea2a90d83, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/28217449ec624cd993a1407c135d71e7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/2852965ed1cc410bba4d41c2544c89b3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5e47fbe26af94972a738c203797dd048, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f7f58728136b4dad9cf81bcc2353f0b6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/c35e2c889e4e49fa891dbd892f55e4e6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f4d696b2c2f44a66bd0b7156a66fb1e3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/35a28c5a97f4419b8c9e1aa2af9f2a40, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/4e3f2fa2df3c46368483cffa4d74fd97, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/d45f11064ec8425d80eebeb713fade67, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/7682cecbf65d4229bdd0471944b69656, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73a260e030eb43859c3dcce9145519fe, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fea1e4d480af4378bb9f865f396bf9c3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fe0b88b1cc614ea586a94c30906a5117, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/69c255c845844e5c82b9ba22a2237876, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5ca7156704b74e59a09d4981bb42fdbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/58abe2c3c2e1415ea9b45fa871834b18, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/11157146801d4e539bdcc8411e632c18, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/cd85530b014543e6b5fc27bc66675dc0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e4b3a1222d5f415e889756592ba8d882, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/49cefcc392af4b1493070032379b7054, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/33babd009f1a43798716e6a3d0ea7fd0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/9b9f35889c5d47cc918a37429d84d352] to archive 2024-12-06T14:43:43,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:43:43,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/ea2e46c1750547e385135b47b432a618 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/ea2e46c1750547e385135b47b432a618 2024-12-06T14:43:43,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73182c8e24a5479da06a6efd6f745b25 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73182c8e24a5479da06a6efd6f745b25 2024-12-06T14:43:43,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e1e006d7a3114e949114ccaa830b793c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e1e006d7a3114e949114ccaa830b793c 2024-12-06T14:43:43,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/a33e6a24022345e9a5f7f2cea2a90d83 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/a33e6a24022345e9a5f7f2cea2a90d83 2024-12-06T14:43:43,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/28217449ec624cd993a1407c135d71e7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/28217449ec624cd993a1407c135d71e7 2024-12-06T14:43:43,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/2852965ed1cc410bba4d41c2544c89b3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/2852965ed1cc410bba4d41c2544c89b3 2024-12-06T14:43:43,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5e47fbe26af94972a738c203797dd048 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5e47fbe26af94972a738c203797dd048 2024-12-06T14:43:43,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f7f58728136b4dad9cf81bcc2353f0b6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f7f58728136b4dad9cf81bcc2353f0b6 2024-12-06T14:43:43,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/c35e2c889e4e49fa891dbd892f55e4e6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/c35e2c889e4e49fa891dbd892f55e4e6 2024-12-06T14:43:43,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f4d696b2c2f44a66bd0b7156a66fb1e3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/f4d696b2c2f44a66bd0b7156a66fb1e3 2024-12-06T14:43:43,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/35a28c5a97f4419b8c9e1aa2af9f2a40 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/35a28c5a97f4419b8c9e1aa2af9f2a40 2024-12-06T14:43:43,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/4e3f2fa2df3c46368483cffa4d74fd97 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/4e3f2fa2df3c46368483cffa4d74fd97 2024-12-06T14:43:43,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/d45f11064ec8425d80eebeb713fade67 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/d45f11064ec8425d80eebeb713fade67 2024-12-06T14:43:43,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/7682cecbf65d4229bdd0471944b69656 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/7682cecbf65d4229bdd0471944b69656 2024-12-06T14:43:43,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73a260e030eb43859c3dcce9145519fe to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/73a260e030eb43859c3dcce9145519fe 2024-12-06T14:43:43,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fea1e4d480af4378bb9f865f396bf9c3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fea1e4d480af4378bb9f865f396bf9c3 2024-12-06T14:43:43,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fe0b88b1cc614ea586a94c30906a5117 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/fe0b88b1cc614ea586a94c30906a5117 2024-12-06T14:43:43,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/69c255c845844e5c82b9ba22a2237876 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/69c255c845844e5c82b9ba22a2237876 2024-12-06T14:43:43,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5ca7156704b74e59a09d4981bb42fdbf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/5ca7156704b74e59a09d4981bb42fdbf 2024-12-06T14:43:43,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/58abe2c3c2e1415ea9b45fa871834b18 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/58abe2c3c2e1415ea9b45fa871834b18 2024-12-06T14:43:43,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/11157146801d4e539bdcc8411e632c18 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/11157146801d4e539bdcc8411e632c18 2024-12-06T14:43:43,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/cd85530b014543e6b5fc27bc66675dc0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/cd85530b014543e6b5fc27bc66675dc0 2024-12-06T14:43:43,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e4b3a1222d5f415e889756592ba8d882 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/e4b3a1222d5f415e889756592ba8d882 2024-12-06T14:43:43,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/49cefcc392af4b1493070032379b7054 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/49cefcc392af4b1493070032379b7054 2024-12-06T14:43:43,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/33babd009f1a43798716e6a3d0ea7fd0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/33babd009f1a43798716e6a3d0ea7fd0 2024-12-06T14:43:43,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/9b9f35889c5d47cc918a37429d84d352 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/9b9f35889c5d47cc918a37429d84d352 2024-12-06T14:43:44,210 INFO [Thread-1414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1414. 2024-12-06T14:43:44,210 DEBUG [Thread-1414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:43:44,210 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:44,210 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:44,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 206 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 181 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 216 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1280 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3840 rows 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1282 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3846 rows 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1298 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3887 rows 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1284 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3852 rows 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1277 2024-12-06T14:43:44,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3831 rows 2024-12-06T14:43:44,210 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:43:44,210 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:44,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:44,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:44,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:44,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T14:43:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T14:43:44,214 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496224214"}]},"ts":"1733496224214"} 2024-12-06T14:43:44,215 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T14:43:44,215 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T14:43:44,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:43:44,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, UNASSIGN}] 2024-12-06T14:43:44,218 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, UNASSIGN 2024-12-06T14:43:44,218 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:44,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, UNASSIGN because future has completed 2024-12-06T14:43:44,220 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:43:44,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:43:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T14:43:44,372 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:44,372 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:43:44,372 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 415d8e5bcca7a5e3eb923a4249401464, disabling compactions & flushes 2024-12-06T14:43:44,372 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:44,372 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:44,372 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. after waiting 0 ms 2024-12-06T14:43:44,372 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:44,372 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 415d8e5bcca7a5e3eb923a4249401464 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T14:43:44,373 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=A 2024-12-06T14:43:44,373 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:44,373 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=B 2024-12-06T14:43:44,373 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:44,373 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 415d8e5bcca7a5e3eb923a4249401464, store=C 2024-12-06T14:43:44,373 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:44,379 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120699954c53704e43b78e86392b1eea0ce8_415d8e5bcca7a5e3eb923a4249401464 is 50, key is test_row_0/A:col10/1733496224209/Put/seqid=0 2024-12-06T14:43:44,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742243_1419 (size=9914) 2024-12-06T14:43:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T14:43:44,784 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:44,790 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120699954c53704e43b78e86392b1eea0ce8_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120699954c53704e43b78e86392b1eea0ce8_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:44,791 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/714d0f9175334169b2a05234b2337a7c, store: [table=TestAcidGuarantees family=A region=415d8e5bcca7a5e3eb923a4249401464] 2024-12-06T14:43:44,792 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/714d0f9175334169b2a05234b2337a7c is 175, key is test_row_0/A:col10/1733496224209/Put/seqid=0 2024-12-06T14:43:44,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742244_1420 (size=22561) 2024-12-06T14:43:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T14:43:45,199 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=891, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/714d0f9175334169b2a05234b2337a7c 2024-12-06T14:43:45,210 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/b9f1d82ea7c44eb2892de853ee6c156d is 50, key is test_row_0/B:col10/1733496224209/Put/seqid=0 2024-12-06T14:43:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742245_1421 (size=9857) 2024-12-06T14:43:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T14:43:45,614 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=891 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/b9f1d82ea7c44eb2892de853ee6c156d 2024-12-06T14:43:45,621 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/714796268ebd434287350ca2a12dcaf8 is 50, key is test_row_0/C:col10/1733496224209/Put/seqid=0 2024-12-06T14:43:45,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742246_1422 (size=9857) 2024-12-06T14:43:45,626 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=891 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/714796268ebd434287350ca2a12dcaf8 2024-12-06T14:43:45,631 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/A/714d0f9175334169b2a05234b2337a7c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/714d0f9175334169b2a05234b2337a7c 2024-12-06T14:43:45,635 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/714d0f9175334169b2a05234b2337a7c, entries=100, sequenceid=891, filesize=22.0 K 2024-12-06T14:43:45,635 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/B/b9f1d82ea7c44eb2892de853ee6c156d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b9f1d82ea7c44eb2892de853ee6c156d 2024-12-06T14:43:45,638 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b9f1d82ea7c44eb2892de853ee6c156d, entries=100, sequenceid=891, filesize=9.6 K 2024-12-06T14:43:45,639 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/.tmp/C/714796268ebd434287350ca2a12dcaf8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/714796268ebd434287350ca2a12dcaf8 2024-12-06T14:43:45,642 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/714796268ebd434287350ca2a12dcaf8, entries=100, sequenceid=891, filesize=9.6 K 2024-12-06T14:43:45,643 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 415d8e5bcca7a5e3eb923a4249401464 in 1271ms, sequenceid=891, compaction requested=true 2024-12-06T14:43:45,655 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/recovered.edits/894.seqid, newMaxSeqId=894, maxSeqId=4 2024-12-06T14:43:45,655 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. 2024-12-06T14:43:45,656 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 415d8e5bcca7a5e3eb923a4249401464: Waiting for close lock at 1733496224372Running coprocessor pre-close hooks at 1733496224372Disabling compacts and flushes for region at 1733496224372Disabling writes for close at 1733496224372Obtaining lock to block concurrent updates at 1733496224372Preparing flush snapshotting stores in 415d8e5bcca7a5e3eb923a4249401464 at 1733496224372Finished memstore snapshotting TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733496224373 (+1 ms)Flushing stores of TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464. at 1733496224373Flushing 415d8e5bcca7a5e3eb923a4249401464/A: creating writer at 1733496224374 (+1 ms)Flushing mob file 415d8e5bcca7a5e3eb923a4249401464/A: appending metadata at 1733496224379 (+5 ms)Flushing mob file 415d8e5bcca7a5e3eb923a4249401464/A: closing flushed file at 1733496224379Flushing 415d8e5bcca7a5e3eb923a4249401464/A: appending metadata at 1733496224792 (+413 ms)Flushing 415d8e5bcca7a5e3eb923a4249401464/A: closing flushed file at 1733496224792Flushing 415d8e5bcca7a5e3eb923a4249401464/B: creating writer at 1733496225205 (+413 ms)Flushing 415d8e5bcca7a5e3eb923a4249401464/B: appending metadata at 1733496225209 (+4 ms)Flushing 415d8e5bcca7a5e3eb923a4249401464/B: closing flushed file at 1733496225209Flushing 415d8e5bcca7a5e3eb923a4249401464/C: creating writer at 1733496225618 (+409 ms)Flushing 415d8e5bcca7a5e3eb923a4249401464/C: appending metadata at 1733496225621 (+3 ms)Flushing 415d8e5bcca7a5e3eb923a4249401464/C: closing flushed file at 1733496225621Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e71f398: reopening flushed file at 1733496225631 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60476cdc: reopening flushed file at 1733496225635 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54c639b7: reopening flushed file at 1733496225638 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 415d8e5bcca7a5e3eb923a4249401464 in 1271ms, sequenceid=891, compaction requested=true at 1733496225643 (+5 ms)Writing region close event to WAL at 1733496225651 (+8 ms)Running coprocessor post-close hooks at 1733496225655 (+4 ms)Closed at 1733496225655 2024-12-06T14:43:45,658 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:45,658 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=415d8e5bcca7a5e3eb923a4249401464, regionState=CLOSED 2024-12-06T14:43:45,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:43:45,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-06T14:43:45,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 415d8e5bcca7a5e3eb923a4249401464, server=bb7ea3e46d9c,35831,1733496100253 in 1.4400 sec 2024-12-06T14:43:45,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-06T14:43:45,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=415d8e5bcca7a5e3eb923a4249401464, UNASSIGN in 1.4450 sec 2024-12-06T14:43:45,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-06T14:43:45,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4480 sec 2024-12-06T14:43:45,666 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496225666"}]},"ts":"1733496225666"} 2024-12-06T14:43:45,668 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T14:43:45,668 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T14:43:45,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4580 sec 2024-12-06T14:43:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T14:43:46,359 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:46,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T14:43:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,361 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T14:43:46,361 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,363 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,365 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/recovered.edits] 2024-12-06T14:43:46,366 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/35e93b8c52ea46629465e8f1ce5b36bc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/35e93b8c52ea46629465e8f1ce5b36bc 2024-12-06T14:43:46,367 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/714d0f9175334169b2a05234b2337a7c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/714d0f9175334169b2a05234b2337a7c 2024-12-06T14:43:46,368 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7c4f296fee4245be8330b3c1d18eb936 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/A/7c4f296fee4245be8330b3c1d18eb936 2024-12-06T14:43:46,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/79daa967fdcd4f8f81642c2cd8e9efaf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/79daa967fdcd4f8f81642c2cd8e9efaf 2024-12-06T14:43:46,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b9f1d82ea7c44eb2892de853ee6c156d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/b9f1d82ea7c44eb2892de853ee6c156d 2024-12-06T14:43:46,371 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f252e3f1de1e417bb1089c218be84f30 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/B/f252e3f1de1e417bb1089c218be84f30 2024-12-06T14:43:46,373 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/01997f54ed5547e68c2c5999fd286cfe to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/01997f54ed5547e68c2c5999fd286cfe 2024-12-06T14:43:46,373 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/0a3c30f418d94ef5b06b9fc49b382af2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/0a3c30f418d94ef5b06b9fc49b382af2 2024-12-06T14:43:46,374 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/714796268ebd434287350ca2a12dcaf8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/C/714796268ebd434287350ca2a12dcaf8 2024-12-06T14:43:46,376 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/recovered.edits/894.seqid to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464/recovered.edits/894.seqid 2024-12-06T14:43:46,376 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,376 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T14:43:46,376 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T14:43:46,377 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T14:43:46,379 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060241b6b7a2544e0b96576a7babb961b3_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060241b6b7a2544e0b96576a7babb961b3_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,380 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120619adee677b5c4898a3b98fc93c647e94_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120619adee677b5c4898a3b98fc93c647e94_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,381 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206357addcfd365484882f6f29a06fb8e6c_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206357addcfd365484882f6f29a06fb8e6c_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,382 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120640d3891f15b14b74a44a8a42f4f2398b_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120640d3891f15b14b74a44a8a42f4f2398b_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,383 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064fd05021057c4608b594942e9f06df61_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064fd05021057c4608b594942e9f06df61_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,384 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206513caca63a6f44d393bbfa2b51b74f98_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206513caca63a6f44d393bbfa2b51b74f98_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,384 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066576d2f1cb3743ff80e720d20697e0b4_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066576d2f1cb3743ff80e720d20697e0b4_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,385 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066a0ee7f644444e859051fc0f51550b41_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066a0ee7f644444e859051fc0f51550b41_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,386 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120686833b12c83244ea9217fea754d55215_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120686833b12c83244ea9217fea754d55215_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,387 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206911500d32a794914a39db173de5051ca_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206911500d32a794914a39db173de5051ca_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,389 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206915685828fe742b1855b68297ce61b1d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206915685828fe742b1855b68297ce61b1d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,390 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206973cb894b8b24dd194c7a11d5949e80d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206973cb894b8b24dd194c7a11d5949e80d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,391 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120699954c53704e43b78e86392b1eea0ce8_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120699954c53704e43b78e86392b1eea0ce8_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,392 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a35c32c70a7648eca3e713e795907a7d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a35c32c70a7648eca3e713e795907a7d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,393 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206adc91ca3b46a4091a882c569b74419e7_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206adc91ca3b46a4091a882c569b74419e7_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,394 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206caf8dc4fca2c47cb935954b831b1faea_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206caf8dc4fca2c47cb935954b831b1faea_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,395 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d096dc2af158431ea53ce66763c6a794_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d096dc2af158431ea53ce66763c6a794_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,396 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dddf6ca10355499d877c8178eafb4037_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dddf6ca10355499d877c8178eafb4037_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,397 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206de9715347296456e9804dd3a7d2a340d_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206de9715347296456e9804dd3a7d2a340d_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,398 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fba626e03ed44e6483e550cf1d97d6cc_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fba626e03ed44e6483e550cf1d97d6cc_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,398 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ff00b2ae4b744fb683cc3fe0df6e5eb6_415d8e5bcca7a5e3eb923a4249401464 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ff00b2ae4b744fb683cc3fe0df6e5eb6_415d8e5bcca7a5e3eb923a4249401464 2024-12-06T14:43:46,399 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T14:43:46,400 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,403 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T14:43:46,405 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T14:43:46,406 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,406 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T14:43:46,406 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733496226406"}]},"ts":"9223372036854775807"} 2024-12-06T14:43:46,408 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T14:43:46,408 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 415d8e5bcca7a5e3eb923a4249401464, NAME => 'TestAcidGuarantees,,1733496191681.415d8e5bcca7a5e3eb923a4249401464.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T14:43:46,408 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T14:43:46,408 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733496226408"}]},"ts":"9223372036854775807"} 2024-12-06T14:43:46,410 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T14:43:46,411 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-12-06T14:43:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T14:43:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T14:43:46,472 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:46,480 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=577 (was 585), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 5596) - AvailableMemoryMB LEAK? - 2024-12-06T14:43:46,491 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=577, ProcessCount=11, AvailableMemoryMB=6842 2024-12-06T14:43:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:43:46,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T14:43:46,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:43:46,495 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:46,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-12-06T14:43:46,496 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:43:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-06T14:43:46,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742247_1423 (size=1061) 2024-12-06T14:43:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-06T14:43:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-06T14:43:46,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:43:46,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742248_1424 (size=53) 2024-12-06T14:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-06T14:43:47,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:47,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e75a56a757f742066bce3d3e372aff1b, disabling compactions & flushes 2024-12-06T14:43:47,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. after waiting 0 ms 2024-12-06T14:43:47,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e75a56a757f742066bce3d3e372aff1b: Waiting for close lock at 1733496227313Disabling compacts and flushes for region at 1733496227313Disabling writes for close at 1733496227313Writing region close event to WAL at 1733496227313Closed at 1733496227313 2024-12-06T14:43:47,314 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:43:47,314 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733496227314"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496227314"}]},"ts":"1733496227314"} 2024-12-06T14:43:47,316 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:43:47,316 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:43:47,316 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496227316"}]},"ts":"1733496227316"} 2024-12-06T14:43:47,318 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T14:43:47,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, ASSIGN}] 2024-12-06T14:43:47,319 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, ASSIGN 2024-12-06T14:43:47,320 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=false 2024-12-06T14:43:47,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=e75a56a757f742066bce3d3e372aff1b, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:47,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, ASSIGN because future has completed 2024-12-06T14:43:47,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:43:47,626 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,626 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:47,627 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,627 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:47,627 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,627 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-06T14:43:47,628 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,629 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:47,629 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e75a56a757f742066bce3d3e372aff1b columnFamilyName A 2024-12-06T14:43:47,629 DEBUG [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:47,630 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.HStore(327): Store=e75a56a757f742066bce3d3e372aff1b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:47,630 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,631 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:47,631 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e75a56a757f742066bce3d3e372aff1b columnFamilyName B 2024-12-06T14:43:47,631 DEBUG [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:47,631 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.HStore(327): Store=e75a56a757f742066bce3d3e372aff1b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:47,631 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,632 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:43:47,632 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e75a56a757f742066bce3d3e372aff1b columnFamilyName C 2024-12-06T14:43:47,632 DEBUG [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:47,633 INFO [StoreOpener-e75a56a757f742066bce3d3e372aff1b-1 {}] regionserver.HStore(327): Store=e75a56a757f742066bce3d3e372aff1b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:47,633 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,633 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,634 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,634 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,635 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,635 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,636 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:43:47,637 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,639 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:47,640 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened e75a56a757f742066bce3d3e372aff1b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70287674, jitterRate=0.04736796021461487}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:43:47,640 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:47,640 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for e75a56a757f742066bce3d3e372aff1b: Running coprocessor pre-open hook at 1733496227627Writing region info on filesystem at 1733496227627Initializing all the Stores at 1733496227628 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496227628Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496227628Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496227628Cleaning up temporary data from old regions at 1733496227635 (+7 ms)Running coprocessor post-open hooks at 1733496227640 (+5 ms)Region opened successfully at 1733496227640 2024-12-06T14:43:47,641 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., pid=116, masterSystemTime=1733496227623 2024-12-06T14:43:47,643 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,643 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:47,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=e75a56a757f742066bce3d3e372aff1b, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:47,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:43:47,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-06T14:43:47,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 in 174 msec 2024-12-06T14:43:47,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-06T14:43:47,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, ASSIGN in 330 msec 2024-12-06T14:43:47,651 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:43:47,652 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496227651"}]},"ts":"1733496227651"} 2024-12-06T14:43:47,653 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T14:43:47,655 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:43:47,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-12-06T14:43:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-06T14:43:48,649 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:48,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e1a424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,651 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@548c9cfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,652 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4566a44e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:43:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:43:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@433fed48 2024-12-06T14:43:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:43:48,656 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:43:48,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407ebc73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51cd2ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19936346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@167d0689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3210aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a99e37e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc7b8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a0647a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12efde8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c49f49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8e554b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb46e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b569ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2446cad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6449638c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5add2189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,678 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a041ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52074950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,681 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2261c04c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,682 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b63ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:43:48,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:48,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:43:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:43:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14561854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:43:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:48,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f19f6bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:48,686 DEBUG [Thread-1911 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,686 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,686 DEBUG [Thread-1915 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,686 DEBUG [Thread-1922 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:48,687 DEBUG [Thread-1917 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,687 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-06T14:43:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,688 DEBUG [Thread-1924 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,689 DEBUG [Thread-1913 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:48,690 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:48,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,695 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,696 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:43:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:48,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:48,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:43:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2] 2024-12-06T14:43:48,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:48,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:48,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:48,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:48,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:48,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:48,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ebf87e7ce76a413ba73a80450a518798 is 50, key is test_row_0/A:col10/1733496228701/Put/seqid=0 2024-12-06T14:43:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496288748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496288750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496288751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742249_1425 (size=16681) 2024-12-06T14:43:48,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ebf87e7ce76a413ba73a80450a518798 2024-12-06T14:43:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496288761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496288758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/043e4270e53d4277ae5f68f099206ee2 is 50, key is test_row_0/B:col10/1733496228701/Put/seqid=0 2024-12-06T14:43:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:48,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742250_1426 (size=12001) 2024-12-06T14:43:48,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:48,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:48,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:48,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:48,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:48,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496288868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496288868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496288869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496288878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496288878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:49,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496289079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496289080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496289079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496289090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496289088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/043e4270e53d4277ae5f68f099206ee2 2024-12-06T14:43:49,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/5f2937eed8064b70a058bffa529bff02 is 50, key is test_row_0/C:col10/1733496228701/Put/seqid=0 2024-12-06T14:43:49,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742251_1427 (size=12001) 2024-12-06T14:43:49,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:49,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:49,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:49,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496289398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496289398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496289398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496289398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496289398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:49,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:49,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/5f2937eed8064b70a058bffa529bff02 2024-12-06T14:43:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ebf87e7ce76a413ba73a80450a518798 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ebf87e7ce76a413ba73a80450a518798 2024-12-06T14:43:49,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ebf87e7ce76a413ba73a80450a518798, entries=250, sequenceid=14, filesize=16.3 K 2024-12-06T14:43:49,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/043e4270e53d4277ae5f68f099206ee2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/043e4270e53d4277ae5f68f099206ee2 2024-12-06T14:43:49,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/043e4270e53d4277ae5f68f099206ee2, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:43:49,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/5f2937eed8064b70a058bffa529bff02 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/5f2937eed8064b70a058bffa529bff02 2024-12-06T14:43:49,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/5f2937eed8064b70a058bffa529bff02, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:43:49,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e75a56a757f742066bce3d3e372aff1b in 985ms, sequenceid=14, compaction requested=false 2024-12-06T14:43:49,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:49,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:49,785 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/8e746ba3ad9142f1bf243a874366ceb4 is 50, key is test_row_0/A:col10/1733496228750/Put/seqid=0 2024-12-06T14:43:49,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742252_1428 (size=12001) 2024-12-06T14:43:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:49,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496289925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496289953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496289953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496289953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496289954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496290058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496290069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496290069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496290069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496290069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,196 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/8e746ba3ad9142f1bf243a874366ceb4 2024-12-06T14:43:50,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/25197219242b4c688d7b0b6d67a961a8 is 50, key is test_row_0/B:col10/1733496228750/Put/seqid=0 2024-12-06T14:43:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742253_1429 (size=12001) 2024-12-06T14:43:50,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496290268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496290278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496290279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496290280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496290280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496290588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496290598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496290598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496290598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496290598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:50,624 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/25197219242b4c688d7b0b6d67a961a8 2024-12-06T14:43:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/24b733f0758743f48f72ab11147fa317 is 50, key is test_row_0/C:col10/1733496228750/Put/seqid=0 2024-12-06T14:43:50,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742254_1430 (size=12001) 2024-12-06T14:43:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:51,033 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/24b733f0758743f48f72ab11147fa317 2024-12-06T14:43:51,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/8e746ba3ad9142f1bf243a874366ceb4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/8e746ba3ad9142f1bf243a874366ceb4 2024-12-06T14:43:51,042 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/8e746ba3ad9142f1bf243a874366ceb4, entries=150, sequenceid=33, filesize=11.7 K 2024-12-06T14:43:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/25197219242b4c688d7b0b6d67a961a8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/25197219242b4c688d7b0b6d67a961a8 2024-12-06T14:43:51,047 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/25197219242b4c688d7b0b6d67a961a8, entries=150, sequenceid=33, filesize=11.7 K 2024-12-06T14:43:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/24b733f0758743f48f72ab11147fa317 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/24b733f0758743f48f72ab11147fa317 2024-12-06T14:43:51,060 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/24b733f0758743f48f72ab11147fa317, entries=150, sequenceid=33, filesize=11.7 K 2024-12-06T14:43:51,061 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e75a56a757f742066bce3d3e372aff1b in 1275ms, sequenceid=33, compaction requested=false 2024-12-06T14:43:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-06T14:43:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-06T14:43:51,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-06T14:43:51,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-12-06T14:43:51,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.3790 sec 2024-12-06T14:43:51,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T14:43:51,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:51,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:51,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:51,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:51,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:51,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:51,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/fb3ed7368a46489183767815713cd60a is 50, key is test_row_0/A:col10/1733496229924/Put/seqid=0 2024-12-06T14:43:51,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742255_1431 (size=14341) 2024-12-06T14:43:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496291130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496291129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496291141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496291142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496291143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,175 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:43:51,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496291248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496291248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496291259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496291259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496291268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496291458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496291458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496291468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496291468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496291478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/fb3ed7368a46489183767815713cd60a 2024-12-06T14:43:51,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/17a06fda06524714bdd407111d225a21 is 50, key is test_row_0/B:col10/1733496229924/Put/seqid=0 2024-12-06T14:43:51,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742256_1432 (size=12001) 2024-12-06T14:43:51,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496291768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496291768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496291778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496291778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496291788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:51,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/17a06fda06524714bdd407111d225a21 2024-12-06T14:43:51,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/ac51fdfc6b2f45f086a858b2bb4be6cd is 50, key is test_row_0/C:col10/1733496229924/Put/seqid=0 2024-12-06T14:43:51,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742257_1433 (size=12001) 2024-12-06T14:43:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496292278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496292278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496292288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:52,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496292288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496292309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:52,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/ac51fdfc6b2f45f086a858b2bb4be6cd 2024-12-06T14:43:52,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/fb3ed7368a46489183767815713cd60a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/fb3ed7368a46489183767815713cd60a 2024-12-06T14:43:52,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/fb3ed7368a46489183767815713cd60a, entries=200, sequenceid=47, filesize=14.0 K 2024-12-06T14:43:52,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/17a06fda06524714bdd407111d225a21 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/17a06fda06524714bdd407111d225a21 2024-12-06T14:43:52,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/17a06fda06524714bdd407111d225a21, entries=150, sequenceid=47, filesize=11.7 K 2024-12-06T14:43:52,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/ac51fdfc6b2f45f086a858b2bb4be6cd as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ac51fdfc6b2f45f086a858b2bb4be6cd 2024-12-06T14:43:52,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ac51fdfc6b2f45f086a858b2bb4be6cd, entries=150, sequenceid=47, filesize=11.7 K 2024-12-06T14:43:52,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e75a56a757f742066bce3d3e372aff1b in 1251ms, sequenceid=47, compaction requested=true 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:52,354 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:52,354 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:43:52,355 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:52,355 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:52,355 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ebf87e7ce76a413ba73a80450a518798, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/8e746ba3ad9142f1bf243a874366ceb4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/fb3ed7368a46489183767815713cd60a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=42.0 K 2024-12-06T14:43:52,355 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/043e4270e53d4277ae5f68f099206ee2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/25197219242b4c688d7b0b6d67a961a8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/17a06fda06524714bdd407111d225a21] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.2 K 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf87e7ce76a413ba73a80450a518798, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496228696 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 043e4270e53d4277ae5f68f099206ee2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496228696 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 25197219242b4c688d7b0b6d67a961a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733496228749 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e746ba3ad9142f1bf243a874366ceb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733496228749 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a06fda06524714bdd407111d225a21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733496229924 2024-12-06T14:43:52,355 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb3ed7368a46489183767815713cd60a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733496229924 2024-12-06T14:43:52,362 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:52,362 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:52,362 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/9b8a622c20cb42119adee60d490234d4 is 50, key is test_row_0/B:col10/1733496229924/Put/seqid=0 2024-12-06T14:43:52,362 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ed371c0123ef40cf848659518654b34e is 50, key is test_row_0/A:col10/1733496229924/Put/seqid=0 2024-12-06T14:43:52,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742258_1434 (size=12104) 2024-12-06T14:43:52,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742259_1435 (size=12104) 2024-12-06T14:43:52,371 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ed371c0123ef40cf848659518654b34e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ed371c0123ef40cf848659518654b34e 2024-12-06T14:43:52,374 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into ed371c0123ef40cf848659518654b34e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:52,374 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:52,374 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=13, startTime=1733496232354; duration=0sec 2024-12-06T14:43:52,374 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:52,374 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:43:52,374 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:52,375 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:52,375 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:43:52,375 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:52,375 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/5f2937eed8064b70a058bffa529bff02, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/24b733f0758743f48f72ab11147fa317, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ac51fdfc6b2f45f086a858b2bb4be6cd] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.2 K 2024-12-06T14:43:52,375 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f2937eed8064b70a058bffa529bff02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496228696 2024-12-06T14:43:52,375 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b733f0758743f48f72ab11147fa317, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733496228749 2024-12-06T14:43:52,376 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac51fdfc6b2f45f086a858b2bb4be6cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733496229924 2024-12-06T14:43:52,381 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:52,382 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/cb42bc18945741749d882c420ae68719 is 50, key is test_row_0/C:col10/1733496229924/Put/seqid=0 2024-12-06T14:43:52,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742260_1436 (size=12104) 2024-12-06T14:43:52,770 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/9b8a622c20cb42119adee60d490234d4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/9b8a622c20cb42119adee60d490234d4 2024-12-06T14:43:52,774 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into 9b8a622c20cb42119adee60d490234d4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:52,774 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:52,774 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=13, startTime=1733496232354; duration=0sec 2024-12-06T14:43:52,774 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:52,774 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:43:52,796 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/cb42bc18945741749d882c420ae68719 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/cb42bc18945741749d882c420ae68719 2024-12-06T14:43:52,801 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into cb42bc18945741749d882c420ae68719(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:52,801 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:52,801 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=13, startTime=1733496232354; duration=0sec 2024-12-06T14:43:52,801 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:52,801 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:43:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T14:43:52,859 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:52,859 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-06T14:43:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T14:43:52,861 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:52,862 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:52,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T14:43:53,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T14:43:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:53,014 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:43:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/60906399466c4eb68ad7e1184fca09f3 is 50, key is test_row_0/A:col10/1733496231128/Put/seqid=0 2024-12-06T14:43:53,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742261_1437 (size=12001) 2024-12-06T14:43:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T14:43:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:53,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496293310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496293312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496293313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496293314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496293328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,331 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496293418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,423 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/60906399466c4eb68ad7e1184fca09f3 2024-12-06T14:43:53,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/b1317972445749ae84a5a8820381aab8 is 50, key is test_row_0/B:col10/1733496231128/Put/seqid=0 2024-12-06T14:43:53,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496293428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496293428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496293428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742262_1438 (size=12001) 2024-12-06T14:43:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T14:43:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496293628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496293638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496293639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496293639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,838 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/b1317972445749ae84a5a8820381aab8 2024-12-06T14:43:53,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/f675771c8804457490f2885b01c80bbf is 50, key is test_row_0/C:col10/1733496231128/Put/seqid=0 2024-12-06T14:43:53,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742263_1439 (size=12001) 2024-12-06T14:43:53,852 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/f675771c8804457490f2885b01c80bbf 2024-12-06T14:43:53,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/60906399466c4eb68ad7e1184fca09f3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/60906399466c4eb68ad7e1184fca09f3 2024-12-06T14:43:53,861 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/60906399466c4eb68ad7e1184fca09f3, entries=150, sequenceid=68, filesize=11.7 K 2024-12-06T14:43:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/b1317972445749ae84a5a8820381aab8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/b1317972445749ae84a5a8820381aab8 2024-12-06T14:43:53,865 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/b1317972445749ae84a5a8820381aab8, entries=150, sequenceid=68, filesize=11.7 K 2024-12-06T14:43:53,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/f675771c8804457490f2885b01c80bbf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/f675771c8804457490f2885b01c80bbf 2024-12-06T14:43:53,869 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/f675771c8804457490f2885b01c80bbf, entries=150, sequenceid=68, filesize=11.7 K 2024-12-06T14:43:53,870 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e75a56a757f742066bce3d3e372aff1b in 856ms, sequenceid=68, compaction requested=false 2024-12-06T14:43:53,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:53,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:53,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-06T14:43:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-06T14:43:53,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-06T14:43:53,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-12-06T14:43:53,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0180 sec 2024-12-06T14:43:53,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:43:53,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:53,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:53,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:53,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:53,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:53,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:53,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/85bd74fd5575484a876a5eaf2374c2ab is 50, key is test_row_0/A:col10/1733496233309/Put/seqid=0 2024-12-06T14:43:53,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742264_1440 (size=14341) 2024-12-06T14:43:53,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496293977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496293977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496293983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496293983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T14:43:53,998 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:53,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-06T14:43:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T14:43:54,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:54,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:54,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:54,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496294088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496294088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496294098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496294098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T14:43:54,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-06T14:43:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:54,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496294298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496294298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-06T14:43:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:54,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496294308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496294308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T14:43:54,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/85bd74fd5575484a876a5eaf2374c2ab 2024-12-06T14:43:54,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/385b5c33d906459db3d405fab0208e53 is 50, key is test_row_0/B:col10/1733496233309/Put/seqid=0 2024-12-06T14:43:54,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742265_1441 (size=12001) 2024-12-06T14:43:54,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/385b5c33d906459db3d405fab0208e53 2024-12-06T14:43:54,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/c44d9f3432f140cfa0c03a7c64aa46b3 is 50, key is test_row_0/C:col10/1733496233309/Put/seqid=0 2024-12-06T14:43:54,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742266_1442 (size=12001) 2024-12-06T14:43:54,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/c44d9f3432f140cfa0c03a7c64aa46b3 2024-12-06T14:43:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/85bd74fd5575484a876a5eaf2374c2ab as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85bd74fd5575484a876a5eaf2374c2ab 2024-12-06T14:43:54,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85bd74fd5575484a876a5eaf2374c2ab, entries=200, sequenceid=83, filesize=14.0 K 2024-12-06T14:43:54,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/385b5c33d906459db3d405fab0208e53 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/385b5c33d906459db3d405fab0208e53 2024-12-06T14:43:54,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/385b5c33d906459db3d405fab0208e53, entries=150, sequenceid=83, filesize=11.7 K 2024-12-06T14:43:54,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/c44d9f3432f140cfa0c03a7c64aa46b3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/c44d9f3432f140cfa0c03a7c64aa46b3 2024-12-06T14:43:54,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/c44d9f3432f140cfa0c03a7c64aa46b3, entries=150, sequenceid=83, filesize=11.7 K 2024-12-06T14:43:54,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e75a56a757f742066bce3d3e372aff1b in 476ms, sequenceid=83, compaction requested=true 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:54,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:54,420 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:54,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:43:54,421 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,421 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,421 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/9b8a622c20cb42119adee60d490234d4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/b1317972445749ae84a5a8820381aab8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/385b5c33d906459db3d405fab0208e53] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.3 K 2024-12-06T14:43:54,421 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ed371c0123ef40cf848659518654b34e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/60906399466c4eb68ad7e1184fca09f3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85bd74fd5575484a876a5eaf2374c2ab] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=37.5 K 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b8a622c20cb42119adee60d490234d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733496229924 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed371c0123ef40cf848659518654b34e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733496229924 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b1317972445749ae84a5a8820381aab8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733496231128 2024-12-06T14:43:54,421 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60906399466c4eb68ad7e1184fca09f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733496231128 2024-12-06T14:43:54,422 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 385b5c33d906459db3d405fab0208e53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496233309 2024-12-06T14:43:54,422 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bd74fd5575484a876a5eaf2374c2ab, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496233309 2024-12-06T14:43:54,427 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:54,427 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/fb62464619304ad99b3b6b9a041a7924 is 50, key is test_row_0/B:col10/1733496233309/Put/seqid=0 2024-12-06T14:43:54,428 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:54,429 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/1c96c92ab07a4cd5b549410d3dca3399 is 50, key is test_row_0/A:col10/1733496233309/Put/seqid=0 2024-12-06T14:43:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742267_1443 (size=12207) 2024-12-06T14:43:54,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742268_1444 (size=12207) 2024-12-06T14:43:54,441 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/fb62464619304ad99b3b6b9a041a7924 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/fb62464619304ad99b3b6b9a041a7924 2024-12-06T14:43:54,445 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into fb62464619304ad99b3b6b9a041a7924(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:54,445 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:54,445 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=13, startTime=1733496234420; duration=0sec 2024-12-06T14:43:54,445 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:54,445 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:43:54,445 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:54,446 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:54,446 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:43:54,446 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,446 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/cb42bc18945741749d882c420ae68719, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/f675771c8804457490f2885b01c80bbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/c44d9f3432f140cfa0c03a7c64aa46b3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.3 K 2024-12-06T14:43:54,446 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cb42bc18945741749d882c420ae68719, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733496229924 2024-12-06T14:43:54,446 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f675771c8804457490f2885b01c80bbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733496231128 2024-12-06T14:43:54,446 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c44d9f3432f140cfa0c03a7c64aa46b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496233309 2024-12-06T14:43:54,451 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#371 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:54,452 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/93cebf93bce34934baade8e8d67645b4 is 50, key is test_row_0/C:col10/1733496233309/Put/seqid=0 2024-12-06T14:43:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742269_1445 (size=12207) 2024-12-06T14:43:54,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:54,458 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/12f946232aea41e69d5b9967ee197b5f is 50, key is test_row_0/A:col10/1733496233975/Put/seqid=0 2024-12-06T14:43:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742270_1446 (size=12001) 2024-12-06T14:43:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:54,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T14:43:54,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496294634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496294636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496294637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496294638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496294748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496294748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496294748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496294748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,843 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/1c96c92ab07a4cd5b549410d3dca3399 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/1c96c92ab07a4cd5b549410d3dca3399 2024-12-06T14:43:54,846 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into 1c96c92ab07a4cd5b549410d3dca3399(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:54,846 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:54,847 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=13, startTime=1733496234420; duration=0sec 2024-12-06T14:43:54,847 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:54,847 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:43:54,860 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/93cebf93bce34934baade8e8d67645b4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/93cebf93bce34934baade8e8d67645b4 2024-12-06T14:43:54,864 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 93cebf93bce34934baade8e8d67645b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:54,864 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:54,864 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=13, startTime=1733496234420; duration=0sec 2024-12-06T14:43:54,864 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:54,864 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:43:54,870 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/12f946232aea41e69d5b9967ee197b5f 2024-12-06T14:43:54,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/0e655ee236b54a63b9dc16ae33b4cee1 is 50, key is test_row_0/B:col10/1733496233975/Put/seqid=0 2024-12-06T14:43:54,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742271_1447 (size=12001) 2024-12-06T14:43:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496294968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496294969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496294969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496294970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T14:43:55,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496295278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496295278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496295279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496295288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,297 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/0e655ee236b54a63b9dc16ae33b4cee1 2024-12-06T14:43:55,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6e01b3bdb97e4ad3b54f2b45180f519f is 50, key is test_row_0/C:col10/1733496233975/Put/seqid=0 2024-12-06T14:43:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742272_1448 (size=12001) 2024-12-06T14:43:55,314 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6e01b3bdb97e4ad3b54f2b45180f519f 2024-12-06T14:43:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/12f946232aea41e69d5b9967ee197b5f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/12f946232aea41e69d5b9967ee197b5f 2024-12-06T14:43:55,331 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/12f946232aea41e69d5b9967ee197b5f, entries=150, sequenceid=101, filesize=11.7 K 2024-12-06T14:43:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/0e655ee236b54a63b9dc16ae33b4cee1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0e655ee236b54a63b9dc16ae33b4cee1 2024-12-06T14:43:55,334 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0e655ee236b54a63b9dc16ae33b4cee1, entries=150, sequenceid=101, filesize=11.7 K 2024-12-06T14:43:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6e01b3bdb97e4ad3b54f2b45180f519f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e01b3bdb97e4ad3b54f2b45180f519f 2024-12-06T14:43:55,337 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e01b3bdb97e4ad3b54f2b45180f519f, entries=150, sequenceid=101, filesize=11.7 K 2024-12-06T14:43:55,338 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e75a56a757f742066bce3d3e372aff1b in 880ms, sequenceid=101, compaction requested=false 2024-12-06T14:43:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-06T14:43:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-06T14:43:55,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-06T14:43:55,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3380 sec 2024-12-06T14:43:55,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3420 sec 2024-12-06T14:43:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:55,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T14:43:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:55,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ba845d4ed7a34673abfb046ccc71f9ab is 50, key is test_row_0/A:col10/1733496234635/Put/seqid=0 2024-12-06T14:43:55,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742273_1449 (size=14341) 2024-12-06T14:43:55,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ba845d4ed7a34673abfb046ccc71f9ab 2024-12-06T14:43:55,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/d8ff0dc40b6949268115b7e2b624dd33 is 50, key is test_row_0/B:col10/1733496234635/Put/seqid=0 2024-12-06T14:43:55,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742274_1450 (size=12001) 2024-12-06T14:43:55,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/d8ff0dc40b6949268115b7e2b624dd33 2024-12-06T14:43:55,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6cfb820348a04db2a00bbeeaddcdf6a0 is 50, key is test_row_0/C:col10/1733496234635/Put/seqid=0 2024-12-06T14:43:55,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742275_1451 (size=12001) 2024-12-06T14:43:55,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6cfb820348a04db2a00bbeeaddcdf6a0 2024-12-06T14:43:55,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ba845d4ed7a34673abfb046ccc71f9ab as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ba845d4ed7a34673abfb046ccc71f9ab 2024-12-06T14:43:55,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ba845d4ed7a34673abfb046ccc71f9ab, entries=200, sequenceid=119, filesize=14.0 K 2024-12-06T14:43:55,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/d8ff0dc40b6949268115b7e2b624dd33 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/d8ff0dc40b6949268115b7e2b624dd33 2024-12-06T14:43:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/d8ff0dc40b6949268115b7e2b624dd33, entries=150, sequenceid=119, filesize=11.7 K 2024-12-06T14:43:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6cfb820348a04db2a00bbeeaddcdf6a0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6cfb820348a04db2a00bbeeaddcdf6a0 2024-12-06T14:43:55,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6cfb820348a04db2a00bbeeaddcdf6a0, entries=150, sequenceid=119, filesize=11.7 K 2024-12-06T14:43:55,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=46.96 KB/48090 for e75a56a757f742066bce3d3e372aff1b in 39ms, sequenceid=119, compaction requested=true 2024-12-06T14:43:55,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:55,390 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:55,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:55,390 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:55,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:55,390 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:43:55,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:43:55,390 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:55,390 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:55,390 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/fb62464619304ad99b3b6b9a041a7924, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0e655ee236b54a63b9dc16ae33b4cee1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/d8ff0dc40b6949268115b7e2b624dd33] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.4 K 2024-12-06T14:43:55,390 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/1c96c92ab07a4cd5b549410d3dca3399, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/12f946232aea41e69d5b9967ee197b5f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ba845d4ed7a34673abfb046ccc71f9ab] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=37.6 K 2024-12-06T14:43:55,391 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c96c92ab07a4cd5b549410d3dca3399, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496233309 2024-12-06T14:43:55,391 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting fb62464619304ad99b3b6b9a041a7924, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496233309 2024-12-06T14:43:55,391 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f946232aea41e69d5b9967ee197b5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733496233975 2024-12-06T14:43:55,391 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e655ee236b54a63b9dc16ae33b4cee1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733496233975 2024-12-06T14:43:55,391 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba845d4ed7a34673abfb046ccc71f9ab, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733496234632 2024-12-06T14:43:55,391 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ff0dc40b6949268115b7e2b624dd33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733496234635 2024-12-06T14:43:55,396 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:55,396 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3401e7314eb6446c943b4d4309048ba4 is 50, key is test_row_0/B:col10/1733496234635/Put/seqid=0 2024-12-06T14:43:55,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T14:43:55,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:55,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:55,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:55,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:55,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:55,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:55,397 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:55,397 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/aeb71507df76471b86af13de76a4b10a is 50, key is test_row_0/A:col10/1733496234635/Put/seqid=0 2024-12-06T14:43:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742276_1452 (size=12309) 2024-12-06T14:43:55,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/db07ff67acdb4988ab05a491639c5491 is 50, key is test_row_0/A:col10/1733496235379/Put/seqid=0 2024-12-06T14:43:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742277_1453 (size=12309) 2024-12-06T14:43:55,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742278_1454 (size=14391) 2024-12-06T14:43:55,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/db07ff67acdb4988ab05a491639c5491 2024-12-06T14:43:55,413 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/aeb71507df76471b86af13de76a4b10a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/aeb71507df76471b86af13de76a4b10a 2024-12-06T14:43:55,418 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into aeb71507df76471b86af13de76a4b10a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:55,418 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:55,418 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=13, startTime=1733496235390; duration=0sec 2024-12-06T14:43:55,418 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:55,418 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:43:55,418 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:55,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/de681a6247f14fc5bec84f607a2b3f10 is 50, key is test_row_0/B:col10/1733496235379/Put/seqid=0 2024-12-06T14:43:55,419 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:55,419 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:43:55,419 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:55,420 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/93cebf93bce34934baade8e8d67645b4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e01b3bdb97e4ad3b54f2b45180f519f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6cfb820348a04db2a00bbeeaddcdf6a0] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.4 K 2024-12-06T14:43:55,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93cebf93bce34934baade8e8d67645b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733496233309 2024-12-06T14:43:55,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e01b3bdb97e4ad3b54f2b45180f519f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733496233975 2024-12-06T14:43:55,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cfb820348a04db2a00bbeeaddcdf6a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733496234635 2024-12-06T14:43:55,441 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:55,442 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/81148de3e6d0479fa3f498123f972b77 is 50, key is test_row_0/C:col10/1733496234635/Put/seqid=0 2024-12-06T14:43:55,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742279_1455 (size=12051) 2024-12-06T14:43:55,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/de681a6247f14fc5bec84f607a2b3f10 2024-12-06T14:43:55,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/a6f301a2d0a1492d87f6b89ecccef09c is 50, key is test_row_0/C:col10/1733496235379/Put/seqid=0 2024-12-06T14:43:55,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742280_1456 (size=12309) 2024-12-06T14:43:55,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742281_1457 (size=12051) 2024-12-06T14:43:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496235467, max timestamp=1733496235548Num uniques 859; 2024-12-06T14:43:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496235467, max timestamp=1733496235548Num uniques 859; 2024-12-06T14:43:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496235467, max timestamp=1733496235548Num uniques 859; 2024-12-06T14:43:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:55,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496295652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496295768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496295788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496295788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496295799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496295799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:55,808 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3401e7314eb6446c943b4d4309048ba4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3401e7314eb6446c943b4d4309048ba4 2024-12-06T14:43:55,816 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into 3401e7314eb6446c943b4d4309048ba4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:55,816 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:55,816 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=13, startTime=1733496235390; duration=0sec 2024-12-06T14:43:55,816 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:55,816 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:43:55,889 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/81148de3e6d0479fa3f498123f972b77 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/81148de3e6d0479fa3f498123f972b77 2024-12-06T14:43:55,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/a6f301a2d0a1492d87f6b89ecccef09c 2024-12-06T14:43:55,892 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 81148de3e6d0479fa3f498123f972b77(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:55,892 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:55,892 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=13, startTime=1733496235390; duration=0sec 2024-12-06T14:43:55,892 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:55,892 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:43:55,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/db07ff67acdb4988ab05a491639c5491 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/db07ff67acdb4988ab05a491639c5491 2024-12-06T14:43:55,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/db07ff67acdb4988ab05a491639c5491, entries=200, sequenceid=130, filesize=14.1 K 2024-12-06T14:43:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/de681a6247f14fc5bec84f607a2b3f10 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/de681a6247f14fc5bec84f607a2b3f10 2024-12-06T14:43:55,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/de681a6247f14fc5bec84f607a2b3f10, entries=150, sequenceid=130, filesize=11.8 K 2024-12-06T14:43:55,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/a6f301a2d0a1492d87f6b89ecccef09c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6f301a2d0a1492d87f6b89ecccef09c 2024-12-06T14:43:55,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6f301a2d0a1492d87f6b89ecccef09c, entries=150, sequenceid=130, filesize=11.8 K 2024-12-06T14:43:55,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e75a56a757f742066bce3d3e372aff1b in 512ms, sequenceid=130, compaction requested=false 2024-12-06T14:43:55,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:55,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-06T14:43:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:43:55,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/6a37ad0688bc4d3781e182e50dc99773 is 50, key is test_row_0/A:col10/1733496235980/Put/seqid=0 2024-12-06T14:43:55,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742282_1458 (size=14541) 2024-12-06T14:43:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496296024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496296138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T14:43:56,149 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:56,149 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-06T14:43:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T14:43:56,150 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:56,151 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:56,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T14:43:56,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496296348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/6a37ad0688bc4d3781e182e50dc99773 2024-12-06T14:43:56,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3e44b5e45fc743969d1ff7864b832444 is 50, key is test_row_0/B:col10/1733496235980/Put/seqid=0 2024-12-06T14:43:56,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742283_1459 (size=12151) 2024-12-06T14:43:56,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:56,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:56,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T14:43:56,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496296668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T14:43:56,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3e44b5e45fc743969d1ff7864b832444 2024-12-06T14:43:56,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/82bffd63ad5e4bc3b0120ff38a292b07 is 50, key is test_row_0/C:col10/1733496235980/Put/seqid=0 2024-12-06T14:43:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496296808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496296808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496296808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,820 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:56,819 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,820 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742284_1460 (size=12151) 2024-12-06T14:43:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496296818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:56,826 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:56,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:56,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496297178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:57,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/82bffd63ad5e4bc3b0120ff38a292b07 2024-12-06T14:43:57,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/6a37ad0688bc4d3781e182e50dc99773 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6a37ad0688bc4d3781e182e50dc99773 2024-12-06T14:43:57,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6a37ad0688bc4d3781e182e50dc99773, entries=200, sequenceid=170, filesize=14.2 K 2024-12-06T14:43:57,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3e44b5e45fc743969d1ff7864b832444 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3e44b5e45fc743969d1ff7864b832444 2024-12-06T14:43:57,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3e44b5e45fc743969d1ff7864b832444, entries=150, sequenceid=170, filesize=11.9 K 2024-12-06T14:43:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/82bffd63ad5e4bc3b0120ff38a292b07 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/82bffd63ad5e4bc3b0120ff38a292b07 2024-12-06T14:43:57,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/82bffd63ad5e4bc3b0120ff38a292b07, entries=150, sequenceid=170, filesize=11.9 K 2024-12-06T14:43:57,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e75a56a757f742066bce3d3e372aff1b in 1261ms, sequenceid=170, compaction requested=true 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:57,243 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:57,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T14:43:57,244 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:57,245 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:57,245 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:43:57,245 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,245 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/aeb71507df76471b86af13de76a4b10a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/db07ff67acdb4988ab05a491639c5491, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6a37ad0688bc4d3781e182e50dc99773] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=40.3 K 2024-12-06T14:43:57,246 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36511 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:57,246 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:43:57,246 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,246 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3401e7314eb6446c943b4d4309048ba4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/de681a6247f14fc5bec84f607a2b3f10, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3e44b5e45fc743969d1ff7864b832444] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.7 K 2024-12-06T14:43:57,246 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb71507df76471b86af13de76a4b10a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733496234635 2024-12-06T14:43:57,246 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3401e7314eb6446c943b4d4309048ba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733496234635 2024-12-06T14:43:57,246 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting db07ff67acdb4988ab05a491639c5491, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733496235373 2024-12-06T14:43:57,247 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting de681a6247f14fc5bec84f607a2b3f10, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733496235379 2024-12-06T14:43:57,247 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a37ad0688bc4d3781e182e50dc99773, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733496235630 2024-12-06T14:43:57,247 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e44b5e45fc743969d1ff7864b832444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733496235630 2024-12-06T14:43:57,259 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:57,259 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/6e2f00c4aa4d45cbbb947a757e1b08ce is 50, key is test_row_0/A:col10/1733496235980/Put/seqid=0 2024-12-06T14:43:57,262 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:57,262 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/ab7673c6011b41e4a55469f1d0428a3b is 50, key is test_row_0/B:col10/1733496235980/Put/seqid=0 2024-12-06T14:43:57,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742285_1461 (size=12561) 2024-12-06T14:43:57,275 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/6e2f00c4aa4d45cbbb947a757e1b08ce as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6e2f00c4aa4d45cbbb947a757e1b08ce 2024-12-06T14:43:57,279 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into 6e2f00c4aa4d45cbbb947a757e1b08ce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:57,279 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:57,279 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=13, startTime=1733496237243; duration=0sec 2024-12-06T14:43:57,279 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:57,279 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:43:57,279 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:57,282 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36511 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:57,282 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:43:57,282 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,282 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/81148de3e6d0479fa3f498123f972b77, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6f301a2d0a1492d87f6b89ecccef09c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/82bffd63ad5e4bc3b0120ff38a292b07] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=35.7 K 2024-12-06T14:43:57,285 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81148de3e6d0479fa3f498123f972b77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733496234635 2024-12-06T14:43:57,285 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f301a2d0a1492d87f6b89ecccef09c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733496235379 2024-12-06T14:43:57,286 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82bffd63ad5e4bc3b0120ff38a292b07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733496235630 2024-12-06T14:43:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T14:43:57,326 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:57,327 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/de64af1f69334eb9b9b25869b06b1958 is 50, key is test_row_0/C:col10/1733496235980/Put/seqid=0 2024-12-06T14:43:57,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742286_1462 (size=12561) 2024-12-06T14:43:57,337 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/ab7673c6011b41e4a55469f1d0428a3b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/ab7673c6011b41e4a55469f1d0428a3b 2024-12-06T14:43:57,341 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into ab7673c6011b41e4a55469f1d0428a3b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:57,341 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:57,341 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=13, startTime=1733496237243; duration=0sec 2024-12-06T14:43:57,341 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:57,341 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:43:57,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742287_1463 (size=12561) 2024-12-06T14:43:57,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T14:43:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,371 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:57,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/e60d8177fe1646a1858f4e5151148e11 is 50, key is test_row_0/A:col10/1733496236018/Put/seqid=0 2024-12-06T14:43:57,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742288_1464 (size=12151) 2024-12-06T14:43:57,386 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/e60d8177fe1646a1858f4e5151148e11 2024-12-06T14:43:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/80296c7c0b7b42828d38ce41ea42761b is 50, key is test_row_0/B:col10/1733496236018/Put/seqid=0 2024-12-06T14:43:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742289_1465 (size=12151) 2024-12-06T14:43:57,430 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/80296c7c0b7b42828d38ce41ea42761b 2024-12-06T14:43:57,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/ecc906c288304be9bb249d440959a52d is 50, key is test_row_0/C:col10/1733496236018/Put/seqid=0 2024-12-06T14:43:57,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742290_1466 (size=12151) 2024-12-06T14:43:57,748 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/de64af1f69334eb9b9b25869b06b1958 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/de64af1f69334eb9b9b25869b06b1958 2024-12-06T14:43:57,753 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into de64af1f69334eb9b9b25869b06b1958(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:57,753 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:57,753 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=13, startTime=1733496237243; duration=0sec 2024-12-06T14:43:57,753 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:57,753 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:43:57,855 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/ecc906c288304be9bb249d440959a52d 2024-12-06T14:43:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/e60d8177fe1646a1858f4e5151148e11 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e60d8177fe1646a1858f4e5151148e11 2024-12-06T14:43:57,861 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e60d8177fe1646a1858f4e5151148e11, entries=150, sequenceid=182, filesize=11.9 K 2024-12-06T14:43:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/80296c7c0b7b42828d38ce41ea42761b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/80296c7c0b7b42828d38ce41ea42761b 2024-12-06T14:43:57,864 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/80296c7c0b7b42828d38ce41ea42761b, entries=150, sequenceid=182, filesize=11.9 K 2024-12-06T14:43:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/ecc906c288304be9bb249d440959a52d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ecc906c288304be9bb249d440959a52d 2024-12-06T14:43:57,869 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ecc906c288304be9bb249d440959a52d, entries=150, sequenceid=182, filesize=11.9 K 2024-12-06T14:43:57,870 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for e75a56a757f742066bce3d3e372aff1b in 499ms, sequenceid=182, compaction requested=false 2024-12-06T14:43:57,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:57,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:57,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-06T14:43:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-06T14:43:57,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-06T14:43:57,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-12-06T14:43:57,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7240 sec 2024-12-06T14:43:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:58,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:58,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/78e1565f6bdd40b1abbfc2d59a0cba7d is 50, key is test_row_0/A:col10/1733496238205/Put/seqid=0 2024-12-06T14:43:58,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742291_1467 (size=9757) 2024-12-06T14:43:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T14:43:58,308 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:43:58,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:43:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-06T14:43:58,310 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:43:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T14:43:58,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:43:58,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:43:58,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496238268, max timestamp=1733496238313Num uniques 859; 2024-12-06T14:43:58,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496238268, max timestamp=1733496238313Num uniques 859; 2024-12-06T14:43:58,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:58,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:58,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496238268, max timestamp=1733496238313Num uniques 859; 2024-12-06T14:43:58,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:43:58,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496298401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T14:43:58,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:58,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:58,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496298508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/78e1565f6bdd40b1abbfc2d59a0cba7d 2024-12-06T14:43:58,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c1fbd8ec0ecc432fa90426eca5493d6c is 50, key is test_row_0/B:col10/1733496238205/Put/seqid=0 2024-12-06T14:43:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T14:43:58,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742292_1468 (size=9757) 2024-12-06T14:43:58,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c1fbd8ec0ecc432fa90426eca5493d6c 2024-12-06T14:43:58,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/96fe59656d004c30959a00ef208f101e is 50, key is test_row_0/C:col10/1733496238205/Put/seqid=0 2024-12-06T14:43:58,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742293_1469 (size=9757) 2024-12-06T14:43:58,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/96fe59656d004c30959a00ef208f101e 2024-12-06T14:43:58,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/78e1565f6bdd40b1abbfc2d59a0cba7d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/78e1565f6bdd40b1abbfc2d59a0cba7d 2024-12-06T14:43:58,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/78e1565f6bdd40b1abbfc2d59a0cba7d, entries=100, sequenceid=193, filesize=9.5 K 2024-12-06T14:43:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c1fbd8ec0ecc432fa90426eca5493d6c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c1fbd8ec0ecc432fa90426eca5493d6c 2024-12-06T14:43:58,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c1fbd8ec0ecc432fa90426eca5493d6c, entries=100, sequenceid=193, filesize=9.5 K 2024-12-06T14:43:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/96fe59656d004c30959a00ef208f101e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/96fe59656d004c30959a00ef208f101e 2024-12-06T14:43:58,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/96fe59656d004c30959a00ef208f101e, entries=100, sequenceid=193, filesize=9.5 K 2024-12-06T14:43:58,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e75a56a757f742066bce3d3e372aff1b in 455ms, sequenceid=193, compaction requested=true 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:58,661 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:58,661 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:43:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:43:58,662 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,662 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,662 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6e2f00c4aa4d45cbbb947a757e1b08ce, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e60d8177fe1646a1858f4e5151148e11, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/78e1565f6bdd40b1abbfc2d59a0cba7d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=33.7 K 2024-12-06T14:43:58,662 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/ab7673c6011b41e4a55469f1d0428a3b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/80296c7c0b7b42828d38ce41ea42761b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c1fbd8ec0ecc432fa90426eca5493d6c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=33.7 K 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2f00c4aa4d45cbbb947a757e1b08ce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733496235630 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ab7673c6011b41e4a55469f1d0428a3b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733496235630 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60d8177fe1646a1858f4e5151148e11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733496235997 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 80296c7c0b7b42828d38ce41ea42761b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733496235997 2024-12-06T14:43:58,662 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e1565f6bdd40b1abbfc2d59a0cba7d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733496238198 2024-12-06T14:43:58,663 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c1fbd8ec0ecc432fa90426eca5493d6c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733496238198 2024-12-06T14:43:58,667 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:58,667 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/20b25252a7a64d8fad663a86abfd6903 is 50, key is test_row_0/A:col10/1733496238205/Put/seqid=0 2024-12-06T14:43:58,670 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:58,670 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f67fca7802ea40c2b6f683680a050515 is 50, key is test_row_0/B:col10/1733496238205/Put/seqid=0 2024-12-06T14:43:58,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742294_1470 (size=12663) 2024-12-06T14:43:58,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742295_1471 (size=12663) 2024-12-06T14:43:58,677 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f67fca7802ea40c2b6f683680a050515 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f67fca7802ea40c2b6f683680a050515 2024-12-06T14:43:58,683 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into f67fca7802ea40c2b6f683680a050515(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:58,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:58,683 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=13, startTime=1733496238661; duration=0sec 2024-12-06T14:43:58,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:43:58,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:43:58,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:43:58,684 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:43:58,684 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:43:58,684 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,684 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/de64af1f69334eb9b9b25869b06b1958, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ecc906c288304be9bb249d440959a52d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/96fe59656d004c30959a00ef208f101e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=33.7 K 2024-12-06T14:43:58,684 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting de64af1f69334eb9b9b25869b06b1958, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733496235630 2024-12-06T14:43:58,684 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc906c288304be9bb249d440959a52d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733496235997 2024-12-06T14:43:58,685 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fe59656d004c30959a00ef208f101e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733496238198 2024-12-06T14:43:58,689 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#398 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:43:58,689 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/7f1d6e98ed43420797b7310f84888601 is 50, key is test_row_0/C:col10/1733496238205/Put/seqid=0 2024-12-06T14:43:58,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742296_1472 (size=12663) 2024-12-06T14:43:58,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-06T14:43:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:43:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:43:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T14:43:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:58,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/868a19f4a6a749b780251d92b18afa79 is 50, key is test_row_0/A:col10/1733496238400/Put/seqid=0 2024-12-06T14:43:58,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742297_1473 (size=14541) 2024-12-06T14:43:58,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/868a19f4a6a749b780251d92b18afa79 2024-12-06T14:43:58,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/413d19c2c3064bb0a8c9495be9ec456b is 50, key is test_row_0/B:col10/1733496238400/Put/seqid=0 2024-12-06T14:43:58,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742298_1474 (size=12151) 2024-12-06T14:43:58,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496298762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496298828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496298829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,836 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:58,836 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496298838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,845 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496298848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,855 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:43:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496298878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:58,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:58,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T14:43:59,077 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/20b25252a7a64d8fad663a86abfd6903 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/20b25252a7a64d8fad663a86abfd6903 2024-12-06T14:43:59,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:59,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:59,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,084 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into 20b25252a7a64d8fad663a86abfd6903(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:59,084 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:59,084 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=13, startTime=1733496238661; duration=0sec 2024-12-06T14:43:59,084 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:59,084 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:43:59,095 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/7f1d6e98ed43420797b7310f84888601 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/7f1d6e98ed43420797b7310f84888601 2024-12-06T14:43:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496299088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:59,100 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 7f1d6e98ed43420797b7310f84888601(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:43:59,100 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:59,100 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=13, startTime=1733496238661; duration=0sec 2024-12-06T14:43:59,100 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:43:59,100 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:43:59,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/413d19c2c3064bb0a8c9495be9ec456b 2024-12-06T14:43:59,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/967365ad34cb4526ba3417dbe9ffd8aa is 50, key is test_row_0/C:col10/1733496238400/Put/seqid=0 2024-12-06T14:43:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742299_1475 (size=12151) 2024-12-06T14:43:59,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496299408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:43:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:43:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:43:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T14:43:59,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:59,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:43:59,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:59,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/967365ad34cb4526ba3417dbe9ffd8aa 2024-12-06T14:43:59,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/868a19f4a6a749b780251d92b18afa79 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/868a19f4a6a749b780251d92b18afa79 2024-12-06T14:43:59,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/868a19f4a6a749b780251d92b18afa79, entries=200, sequenceid=232, filesize=14.2 K 2024-12-06T14:43:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/413d19c2c3064bb0a8c9495be9ec456b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/413d19c2c3064bb0a8c9495be9ec456b 2024-12-06T14:43:59,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/413d19c2c3064bb0a8c9495be9ec456b, entries=150, sequenceid=232, filesize=11.9 K 2024-12-06T14:43:59,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/967365ad34cb4526ba3417dbe9ffd8aa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/967365ad34cb4526ba3417dbe9ffd8aa 2024-12-06T14:43:59,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/967365ad34cb4526ba3417dbe9ffd8aa, entries=150, sequenceid=232, filesize=11.9 K 2024-12-06T14:43:59,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e75a56a757f742066bce3d3e372aff1b in 862ms, sequenceid=232, compaction requested=false 2024-12-06T14:43:59,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:59,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,692 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/7ffd16ca220743d19a14d585592f7a06 is 50, key is test_row_0/A:col10/1733496238752/Put/seqid=0 2024-12-06T14:43:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742300_1476 (size=9757) 2024-12-06T14:43:59,733 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/7ffd16ca220743d19a14d585592f7a06 2024-12-06T14:43:59,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/107078857a06454a9cc583b31837a207 is 50, key is test_row_0/B:col10/1733496238752/Put/seqid=0 2024-12-06T14:43:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742301_1477 (size=9757) 2024-12-06T14:43:59,785 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/107078857a06454a9cc583b31837a207 2024-12-06T14:43:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/e54aa638285641d8b35b0f4924a45ee1 is 50, key is test_row_0/C:col10/1733496238752/Put/seqid=0 2024-12-06T14:43:59,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742302_1478 (size=9757) 2024-12-06T14:43:59,825 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/e54aa638285641d8b35b0f4924a45ee1 2024-12-06T14:43:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/7ffd16ca220743d19a14d585592f7a06 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/7ffd16ca220743d19a14d585592f7a06 2024-12-06T14:43:59,836 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/7ffd16ca220743d19a14d585592f7a06, entries=100, sequenceid=243, filesize=9.5 K 2024-12-06T14:43:59,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/107078857a06454a9cc583b31837a207 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/107078857a06454a9cc583b31837a207 2024-12-06T14:43:59,844 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/107078857a06454a9cc583b31837a207, entries=100, sequenceid=243, filesize=9.5 K 2024-12-06T14:43:59,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/e54aa638285641d8b35b0f4924a45ee1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/e54aa638285641d8b35b0f4924a45ee1 2024-12-06T14:43:59,860 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/e54aa638285641d8b35b0f4924a45ee1, entries=100, sequenceid=243, filesize=9.5 K 2024-12-06T14:43:59,861 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for e75a56a757f742066bce3d3e372aff1b in 169ms, sequenceid=243, compaction requested=true 2024-12-06T14:43:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:43:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:43:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-06T14:43:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-06T14:43:59,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-06T14:43:59,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5520 sec 2024-12-06T14:43:59,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5600 sec 2024-12-06T14:43:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:43:59,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:43:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:43:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:43:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:43:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:43:59,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ab982149335a4900816a509ada63937c is 50, key is test_row_0/A:col10/1733496239967/Put/seqid=0 2024-12-06T14:44:00,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742303_1479 (size=12151) 2024-12-06T14:44:00,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ab982149335a4900816a509ada63937c 2024-12-06T14:44:00,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/110c4dff521b4dc3b9c32d0bda228c69 is 50, key is test_row_0/B:col10/1733496239967/Put/seqid=0 2024-12-06T14:44:00,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742304_1480 (size=12151) 2024-12-06T14:44:00,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/110c4dff521b4dc3b9c32d0bda228c69 2024-12-06T14:44:00,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6e9b7ffbcd1d4604a3f68df128be97a1 is 50, key is test_row_0/C:col10/1733496239967/Put/seqid=0 2024-12-06T14:44:00,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742305_1481 (size=12151) 2024-12-06T14:44:00,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6e9b7ffbcd1d4604a3f68df128be97a1 2024-12-06T14:44:00,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/ab982149335a4900816a509ada63937c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ab982149335a4900816a509ada63937c 2024-12-06T14:44:00,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ab982149335a4900816a509ada63937c, entries=150, sequenceid=253, filesize=11.9 K 2024-12-06T14:44:00,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/110c4dff521b4dc3b9c32d0bda228c69 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/110c4dff521b4dc3b9c32d0bda228c69 2024-12-06T14:44:00,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/110c4dff521b4dc3b9c32d0bda228c69, entries=150, sequenceid=253, filesize=11.9 K 2024-12-06T14:44:00,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/6e9b7ffbcd1d4604a3f68df128be97a1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e9b7ffbcd1d4604a3f68df128be97a1 2024-12-06T14:44:00,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e9b7ffbcd1d4604a3f68df128be97a1, entries=150, sequenceid=253, filesize=11.9 K 2024-12-06T14:44:00,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=67.09 KB/68700 for e75a56a757f742066bce3d3e372aff1b in 103ms, sequenceid=253, compaction requested=true 2024-12-06T14:44:00,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:00,072 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:00,072 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:00,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:00,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:00,078 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49112 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:00,078 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:44:00,078 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,078 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/20b25252a7a64d8fad663a86abfd6903, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/868a19f4a6a749b780251d92b18afa79, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/7ffd16ca220743d19a14d585592f7a06, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ab982149335a4900816a509ada63937c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=48.0 K 2024-12-06T14:44:00,078 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:00,078 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b25252a7a64d8fad663a86abfd6903, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733496235997 2024-12-06T14:44:00,078 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:44:00,078 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T14:44:00,079 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f67fca7802ea40c2b6f683680a050515, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/413d19c2c3064bb0a8c9495be9ec456b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/107078857a06454a9cc583b31837a207, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/110c4dff521b4dc3b9c32d0bda228c69] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=45.6 K 2024-12-06T14:44:00,079 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 868a19f4a6a749b780251d92b18afa79, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733496238353 2024-12-06T14:44:00,079 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f67fca7802ea40c2b6f683680a050515, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733496235997 2024-12-06T14:44:00,079 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 413d19c2c3064bb0a8c9495be9ec456b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733496238353 2024-12-06T14:44:00,079 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ffd16ca220743d19a14d585592f7a06, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733496238752 2024-12-06T14:44:00,079 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 107078857a06454a9cc583b31837a207, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733496238752 2024-12-06T14:44:00,079 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab982149335a4900816a509ada63937c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733496239938 2024-12-06T14:44:00,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:00,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:00,080 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 110c4dff521b4dc3b9c32d0bda228c69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733496239938 2024-12-06T14:44:00,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:00,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:00,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:00,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:00,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/99d063620f61455894233f84647fa02f is 50, key is test_row_0/A:col10/1733496240073/Put/seqid=0 2024-12-06T14:44:00,090 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:00,090 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/90b7f7ca275f49e1a6ac22d1dee5e9dd is 50, key is test_row_0/A:col10/1733496239967/Put/seqid=0 2024-12-06T14:44:00,102 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:00,103 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3d9d397763144fc88f65c220eae8e35f is 50, key is test_row_0/B:col10/1733496239967/Put/seqid=0 2024-12-06T14:44:00,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742306_1482 (size=14741) 2024-12-06T14:44:00,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742307_1483 (size=12799) 2024-12-06T14:44:00,150 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/90b7f7ca275f49e1a6ac22d1dee5e9dd as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/90b7f7ca275f49e1a6ac22d1dee5e9dd 2024-12-06T14:44:00,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742308_1484 (size=12799) 2024-12-06T14:44:00,162 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into 90b7f7ca275f49e1a6ac22d1dee5e9dd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:00,162 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:00,162 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=12, startTime=1733496240071; duration=0sec 2024-12-06T14:44:00,162 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:00,162 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:44:00,162 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:00,163 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/3d9d397763144fc88f65c220eae8e35f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3d9d397763144fc88f65c220eae8e35f 2024-12-06T14:44:00,164 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:00,165 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:44:00,165 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,165 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/7f1d6e98ed43420797b7310f84888601, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/967365ad34cb4526ba3417dbe9ffd8aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/e54aa638285641d8b35b0f4924a45ee1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e9b7ffbcd1d4604a3f68df128be97a1] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=45.6 K 2024-12-06T14:44:00,165 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1d6e98ed43420797b7310f84888601, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733496235997 2024-12-06T14:44:00,166 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 967365ad34cb4526ba3417dbe9ffd8aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733496238353 2024-12-06T14:44:00,166 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting e54aa638285641d8b35b0f4924a45ee1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733496238752 2024-12-06T14:44:00,166 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9b7ffbcd1d4604a3f68df128be97a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733496239938 2024-12-06T14:44:00,168 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into 3d9d397763144fc88f65c220eae8e35f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:00,168 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:00,168 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=12, startTime=1733496240072; duration=0sec 2024-12-06T14:44:00,168 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:00,168 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:44:00,176 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:00,176 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/02ac9ac564784d0ca10f2b2735db7a0b is 50, key is test_row_0/C:col10/1733496239967/Put/seqid=0 2024-12-06T14:44:00,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742309_1485 (size=12799) 2024-12-06T14:44:00,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496300248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:00,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496300358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T14:44:00,458 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:00,459 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-06T14:44:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T14:44:00,460 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:00,461 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:00,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:00,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/99d063620f61455894233f84647fa02f 2024-12-06T14:44:00,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f033751312514b91b5b225b644394e04 is 50, key is test_row_0/B:col10/1733496240073/Put/seqid=0 2024-12-06T14:44:00,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742310_1486 (size=12301) 2024-12-06T14:44:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T14:44:00,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496300568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:00,600 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/02ac9ac564784d0ca10f2b2735db7a0b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/02ac9ac564784d0ca10f2b2735db7a0b 2024-12-06T14:44:00,608 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 02ac9ac564784d0ca10f2b2735db7a0b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:00,609 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:00,609 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=12, startTime=1733496240073; duration=0sec 2024-12-06T14:44:00,609 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:00,609 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:44:00,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T14:44:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:00,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T14:44:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T14:44:00,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496300888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:00,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T14:44:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:00,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:00,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f033751312514b91b5b225b644394e04 2024-12-06T14:44:00,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/caa0d87faccf4343a2fb0859ec22253e is 50, key is test_row_0/C:col10/1733496240073/Put/seqid=0 2024-12-06T14:44:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742311_1487 (size=12301) 2024-12-06T14:44:01,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T14:44:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:01,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T14:44:01,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T14:44:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:01,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:01,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/caa0d87faccf4343a2fb0859ec22253e 2024-12-06T14:44:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/99d063620f61455894233f84647fa02f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/99d063620f61455894233f84647fa02f 2024-12-06T14:44:01,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/99d063620f61455894233f84647fa02f, entries=200, sequenceid=267, filesize=14.4 K 2024-12-06T14:44:01,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f033751312514b91b5b225b644394e04 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f033751312514b91b5b225b644394e04 2024-12-06T14:44:01,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f033751312514b91b5b225b644394e04, entries=150, sequenceid=267, filesize=12.0 K 2024-12-06T14:44:01,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/caa0d87faccf4343a2fb0859ec22253e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/caa0d87faccf4343a2fb0859ec22253e 2024-12-06T14:44:01,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/caa0d87faccf4343a2fb0859ec22253e, entries=150, sequenceid=267, filesize=12.0 K 2024-12-06T14:44:01,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e75a56a757f742066bce3d3e372aff1b in 1294ms, sequenceid=267, compaction requested=false 2024-12-06T14:44:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:01,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T14:44:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:01,381 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:44:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:01,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/591116f71f104a30a4f11cd9a4a4ba63 is 50, key is test_row_0/A:col10/1733496240200/Put/seqid=0 2024-12-06T14:44:01,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742312_1488 (size=12301) 2024-12-06T14:44:01,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:01,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496301473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:01,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:01,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:01,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496301578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T14:44:01,798 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/591116f71f104a30a4f11cd9a4a4ba63 2024-12-06T14:44:01,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496301798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f7a42d193ad64a38beb7652134a94cf1 is 50, key is test_row_0/B:col10/1733496240200/Put/seqid=0 2024-12-06T14:44:01,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742313_1489 (size=12301) 2024-12-06T14:44:02,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496302108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,209 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f7a42d193ad64a38beb7652134a94cf1 2024-12-06T14:44:02,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/d3cac2befb3e4986855143bea706c28b is 50, key is test_row_0/C:col10/1733496240200/Put/seqid=0 2024-12-06T14:44:02,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742314_1490 (size=12301) 2024-12-06T14:44:02,224 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/d3cac2befb3e4986855143bea706c28b 2024-12-06T14:44:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/591116f71f104a30a4f11cd9a4a4ba63 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/591116f71f104a30a4f11cd9a4a4ba63 2024-12-06T14:44:02,231 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/591116f71f104a30a4f11cd9a4a4ba63, entries=150, sequenceid=288, filesize=12.0 K 2024-12-06T14:44:02,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/f7a42d193ad64a38beb7652134a94cf1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f7a42d193ad64a38beb7652134a94cf1 2024-12-06T14:44:02,241 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f7a42d193ad64a38beb7652134a94cf1, entries=150, sequenceid=288, filesize=12.0 K 2024-12-06T14:44:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/d3cac2befb3e4986855143bea706c28b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d3cac2befb3e4986855143bea706c28b 2024-12-06T14:44:02,244 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d3cac2befb3e4986855143bea706c28b, entries=150, sequenceid=288, filesize=12.0 K 2024-12-06T14:44:02,245 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e75a56a757f742066bce3d3e372aff1b in 864ms, sequenceid=288, compaction requested=true 2024-12-06T14:44:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-06T14:44:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-06T14:44:02,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-06T14:44:02,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-12-06T14:44:02,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7890 sec 2024-12-06T14:44:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T14:44:02,619 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:02,619 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-06T14:44:02,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:02,621 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:02,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:02,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:44:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:02,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/a21da74c1aa54fa68d09ad392d71550a is 50, key is test_row_0/A:col10/1733496242632/Put/seqid=0 2024-12-06T14:44:02,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742315_1491 (size=14741) 2024-12-06T14:44:02,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496302696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:02,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:02,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:02,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496302808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55192 deadline: 1733496302858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,862 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55172 deadline: 1733496302868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55204 deadline: 1733496302868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,874 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:02,874 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55218 deadline: 1733496302878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:02,883 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in e75a56a757f742066bce3d3e372aff1b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:02,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:02,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:02,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:02,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:02,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:02,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:03,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496303018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:03,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/a21da74c1aa54fa68d09ad392d71550a 2024-12-06T14:44:03,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/091d268a05fb404bb4877212c9591b79 is 50, key is test_row_0/B:col10/1733496242632/Put/seqid=0 2024-12-06T14:44:03,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742316_1492 (size=12301) 2024-12-06T14:44:03,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:03,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496303328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:03,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:03,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/091d268a05fb404bb4877212c9591b79 2024-12-06T14:44:03,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/581641b1b8124560a2482f0d0887f7fa is 50, key is test_row_0/C:col10/1733496242632/Put/seqid=0 2024-12-06T14:44:03,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742317_1493 (size=12301) 2024-12-06T14:44:03,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:03,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:03,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496303838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:03,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/581641b1b8124560a2482f0d0887f7fa 2024-12-06T14:44:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/a21da74c1aa54fa68d09ad392d71550a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/a21da74c1aa54fa68d09ad392d71550a 2024-12-06T14:44:03,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/a21da74c1aa54fa68d09ad392d71550a, entries=200, sequenceid=303, filesize=14.4 K 2024-12-06T14:44:03,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/091d268a05fb404bb4877212c9591b79 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/091d268a05fb404bb4877212c9591b79 2024-12-06T14:44:03,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/091d268a05fb404bb4877212c9591b79, entries=150, sequenceid=303, filesize=12.0 K 2024-12-06T14:44:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/581641b1b8124560a2482f0d0887f7fa as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/581641b1b8124560a2482f0d0887f7fa 2024-12-06T14:44:03,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/581641b1b8124560a2482f0d0887f7fa, entries=150, sequenceid=303, filesize=12.0 K 2024-12-06T14:44:03,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e75a56a757f742066bce3d3e372aff1b in 1233ms, sequenceid=303, compaction requested=true 2024-12-06T14:44:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:03,866 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:03,866 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:03,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:03,867 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:03,867 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54582 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:03,867 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:44:03,867 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:44:03,867 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,867 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,867 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3d9d397763144fc88f65c220eae8e35f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f033751312514b91b5b225b644394e04, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f7a42d193ad64a38beb7652134a94cf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/091d268a05fb404bb4877212c9591b79] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=48.5 K 2024-12-06T14:44:03,867 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/90b7f7ca275f49e1a6ac22d1dee5e9dd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/99d063620f61455894233f84647fa02f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/591116f71f104a30a4f11cd9a4a4ba63, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/a21da74c1aa54fa68d09ad392d71550a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=53.3 K 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90b7f7ca275f49e1a6ac22d1dee5e9dd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733496239938 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9d397763144fc88f65c220eae8e35f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733496239938 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f033751312514b91b5b225b644394e04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733496240036 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d063620f61455894233f84647fa02f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733496240022 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a42d193ad64a38beb7652134a94cf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733496240200 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591116f71f104a30a4f11cd9a4a4ba63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733496240200 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a21da74c1aa54fa68d09ad392d71550a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733496241420 2024-12-06T14:44:03,868 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 091d268a05fb404bb4877212c9591b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733496241420 2024-12-06T14:44:03,874 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:03,874 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:03,874 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/46adf004f5cd445088548bed9e64cb95 is 50, key is test_row_0/B:col10/1733496242632/Put/seqid=0 2024-12-06T14:44:03,874 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/93e37b0c2f0d403fbd4cba7070fc1e3f is 50, key is test_row_0/A:col10/1733496242632/Put/seqid=0 2024-12-06T14:44:03,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742318_1494 (size=13085) 2024-12-06T14:44:03,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742319_1495 (size=13085) 2024-12-06T14:44:03,880 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/93e37b0c2f0d403fbd4cba7070fc1e3f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/93e37b0c2f0d403fbd4cba7070fc1e3f 2024-12-06T14:44:03,882 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into 93e37b0c2f0d403fbd4cba7070fc1e3f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:03,882 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:03,882 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=12, startTime=1733496243866; duration=0sec 2024-12-06T14:44:03,882 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:03,882 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:44:03,882 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:03,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:03,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:44:03,883 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,883 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/02ac9ac564784d0ca10f2b2735db7a0b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/caa0d87faccf4343a2fb0859ec22253e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d3cac2befb3e4986855143bea706c28b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/581641b1b8124560a2482f0d0887f7fa] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=48.5 K 2024-12-06T14:44:03,883 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ac9ac564784d0ca10f2b2735db7a0b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733496239938 2024-12-06T14:44:03,884 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa0d87faccf4343a2fb0859ec22253e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733496240036 2024-12-06T14:44:03,884 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3cac2befb3e4986855143bea706c28b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733496240200 2024-12-06T14:44:03,884 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581641b1b8124560a2482f0d0887f7fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733496241420 2024-12-06T14:44:03,889 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:03,889 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/474e4193803f4d059ec856c8e2c3bf37 is 50, key is test_row_0/C:col10/1733496242632/Put/seqid=0 2024-12-06T14:44:03,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742320_1496 (size=13085) 2024-12-06T14:44:03,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T14:44:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:03,994 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:44:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:03,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/2c5f62288cfd4409bcb18348e09660ea is 50, key is test_row_0/A:col10/1733496242694/Put/seqid=0 2024-12-06T14:44:04,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742321_1497 (size=12301) 2024-12-06T14:44:04,280 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/46adf004f5cd445088548bed9e64cb95 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/46adf004f5cd445088548bed9e64cb95 2024-12-06T14:44:04,283 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into 46adf004f5cd445088548bed9e64cb95(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:04,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:04,283 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=12, startTime=1733496243866; duration=0sec 2024-12-06T14:44:04,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:04,283 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:44:04,301 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/474e4193803f4d059ec856c8e2c3bf37 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/474e4193803f4d059ec856c8e2c3bf37 2024-12-06T14:44:04,305 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 474e4193803f4d059ec856c8e2c3bf37(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:04,305 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:04,305 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=12, startTime=1733496243866; duration=0sec 2024-12-06T14:44:04,305 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:04,305 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:44:04,402 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/2c5f62288cfd4409bcb18348e09660ea 2024-12-06T14:44:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/0a9dbb9898ab4bb1bee918d05b9fb419 is 50, key is test_row_0/B:col10/1733496242694/Put/seqid=0 2024-12-06T14:44:04,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742322_1498 (size=12301) 2024-12-06T14:44:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:04,813 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/0a9dbb9898ab4bb1bee918d05b9fb419 2024-12-06T14:44:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/a6c6ea83bb5a4531a1e367b7ff950b74 is 50, key is test_row_0/C:col10/1733496242694/Put/seqid=0 2024-12-06T14:44:04,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742323_1499 (size=12301) 2024-12-06T14:44:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:04,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:04,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496304903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:05,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496305008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:05,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496305218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:05,224 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/a6c6ea83bb5a4531a1e367b7ff950b74 2024-12-06T14:44:05,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/2c5f62288cfd4409bcb18348e09660ea as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/2c5f62288cfd4409bcb18348e09660ea 2024-12-06T14:44:05,229 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/2c5f62288cfd4409bcb18348e09660ea, entries=150, sequenceid=321, filesize=12.0 K 2024-12-06T14:44:05,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/0a9dbb9898ab4bb1bee918d05b9fb419 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0a9dbb9898ab4bb1bee918d05b9fb419 2024-12-06T14:44:05,232 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0a9dbb9898ab4bb1bee918d05b9fb419, entries=150, sequenceid=321, filesize=12.0 K 2024-12-06T14:44:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/a6c6ea83bb5a4531a1e367b7ff950b74 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6c6ea83bb5a4531a1e367b7ff950b74 2024-12-06T14:44:05,236 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6c6ea83bb5a4531a1e367b7ff950b74, entries=150, sequenceid=321, filesize=12.0 K 2024-12-06T14:44:05,236 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e75a56a757f742066bce3d3e372aff1b in 1243ms, sequenceid=321, compaction requested=false 2024-12-06T14:44:05,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:05,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:05,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-06T14:44:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-06T14:44:05,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-06T14:44:05,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6160 sec 2024-12-06T14:44:05,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.6200 sec 2024-12-06T14:44:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:05,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T14:44:05,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:05,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:05,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:05,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:05,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:05,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:05,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/d73bf34412f34b83ac41d5840cf0e38a is 50, key is test_row_0/A:col10/1733496244901/Put/seqid=0 2024-12-06T14:44:05,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742324_1500 (size=14741) 2024-12-06T14:44:05,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/d73bf34412f34b83ac41d5840cf0e38a 2024-12-06T14:44:05,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c2e733fc5a2342eba3e4bf4bd3a43dd6 is 50, key is test_row_0/B:col10/1733496244901/Put/seqid=0 2024-12-06T14:44:05,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742325_1501 (size=12301) 2024-12-06T14:44:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496305596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496305708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496305918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:05,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c2e733fc5a2342eba3e4bf4bd3a43dd6 2024-12-06T14:44:05,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/12354ea2a3dd4555afd9d81c40b97e1d is 50, key is test_row_0/C:col10/1733496244901/Put/seqid=0 2024-12-06T14:44:05,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742326_1502 (size=12301) 2024-12-06T14:44:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496306228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:06,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/12354ea2a3dd4555afd9d81c40b97e1d 2024-12-06T14:44:06,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/d73bf34412f34b83ac41d5840cf0e38a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/d73bf34412f34b83ac41d5840cf0e38a 2024-12-06T14:44:06,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/d73bf34412f34b83ac41d5840cf0e38a, entries=200, sequenceid=339, filesize=14.4 K 2024-12-06T14:44:06,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c2e733fc5a2342eba3e4bf4bd3a43dd6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c2e733fc5a2342eba3e4bf4bd3a43dd6 2024-12-06T14:44:06,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c2e733fc5a2342eba3e4bf4bd3a43dd6, entries=150, sequenceid=339, filesize=12.0 K 2024-12-06T14:44:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/12354ea2a3dd4555afd9d81c40b97e1d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/12354ea2a3dd4555afd9d81c40b97e1d 2024-12-06T14:44:06,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/12354ea2a3dd4555afd9d81c40b97e1d, entries=150, sequenceid=339, filesize=12.0 K 2024-12-06T14:44:06,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e75a56a757f742066bce3d3e372aff1b in 837ms, sequenceid=339, compaction requested=true 2024-12-06T14:44:06,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:06,369 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:06,369 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:44:06,370 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:06,370 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:06,370 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/93e37b0c2f0d403fbd4cba7070fc1e3f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/2c5f62288cfd4409bcb18348e09660ea, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/d73bf34412f34b83ac41d5840cf0e38a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=39.2 K 2024-12-06T14:44:06,370 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/46adf004f5cd445088548bed9e64cb95, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0a9dbb9898ab4bb1bee918d05b9fb419, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c2e733fc5a2342eba3e4bf4bd3a43dd6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=36.8 K 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e37b0c2f0d403fbd4cba7070fc1e3f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733496241420 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 46adf004f5cd445088548bed9e64cb95, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733496241420 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c5f62288cfd4409bcb18348e09660ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496242666 2024-12-06T14:44:06,370 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a9dbb9898ab4bb1bee918d05b9fb419, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496242666 2024-12-06T14:44:06,371 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d73bf34412f34b83ac41d5840cf0e38a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496244869 2024-12-06T14:44:06,371 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e733fc5a2342eba3e4bf4bd3a43dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496244890 2024-12-06T14:44:06,375 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:06,375 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/85d6672a1b73493dba0a3eaa0dc785b6 is 50, key is test_row_0/A:col10/1733496244901/Put/seqid=0 2024-12-06T14:44:06,375 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:06,375 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/4a747b10178f4b4382cb78892cdd1bf1 is 50, key is test_row_0/B:col10/1733496244901/Put/seqid=0 2024-12-06T14:44:06,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742328_1504 (size=13187) 2024-12-06T14:44:06,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742327_1503 (size=13187) 2024-12-06T14:44:06,394 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/85d6672a1b73493dba0a3eaa0dc785b6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85d6672a1b73493dba0a3eaa0dc785b6 2024-12-06T14:44:06,398 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into 85d6672a1b73493dba0a3eaa0dc785b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:06,398 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:06,398 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=13, startTime=1733496246368; duration=0sec 2024-12-06T14:44:06,398 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:06,398 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:44:06,398 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:06,399 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:06,399 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:44:06,399 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:06,399 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/474e4193803f4d059ec856c8e2c3bf37, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6c6ea83bb5a4531a1e367b7ff950b74, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/12354ea2a3dd4555afd9d81c40b97e1d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=36.8 K 2024-12-06T14:44:06,399 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 474e4193803f4d059ec856c8e2c3bf37, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733496241420 2024-12-06T14:44:06,399 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c6ea83bb5a4531a1e367b7ff950b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496242666 2024-12-06T14:44:06,399 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12354ea2a3dd4555afd9d81c40b97e1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496244890 2024-12-06T14:44:06,405 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:06,405 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/20f33f1e8c6c4cc6904fba1bc2a83cad is 50, key is test_row_0/C:col10/1733496244901/Put/seqid=0 2024-12-06T14:44:06,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742329_1505 (size=13187) 2024-12-06T14:44:06,416 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/20f33f1e8c6c4cc6904fba1bc2a83cad as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/20f33f1e8c6c4cc6904fba1bc2a83cad 2024-12-06T14:44:06,420 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 20f33f1e8c6c4cc6904fba1bc2a83cad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:06,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:06,420 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=13, startTime=1733496246369; duration=0sec 2024-12-06T14:44:06,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:06,420 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:44:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:06,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:44:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:06,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/de67bf31e1e24c5b8b934d90a4e064b8 is 50, key is test_row_0/A:col10/1733496245595/Put/seqid=0 2024-12-06T14:44:06,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742330_1506 (size=14741) 2024-12-06T14:44:06,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/de67bf31e1e24c5b8b934d90a4e064b8 2024-12-06T14:44:06,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/cc5cdb82b234446bb1772117c9d8c104 is 50, key is test_row_0/B:col10/1733496245595/Put/seqid=0 2024-12-06T14:44:06,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742331_1507 (size=12301) 2024-12-06T14:44:06,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/cc5cdb82b234446bb1772117c9d8c104 2024-12-06T14:44:06,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/63744c9018b84bd4b425cb5c651d351f is 50, key is test_row_0/C:col10/1733496245595/Put/seqid=0 2024-12-06T14:44:06,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742332_1508 (size=12301) 2024-12-06T14:44:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T14:44:06,778 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:06,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-06T14:44:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T14:44:06,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:06,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:06,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:06,784 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/4a747b10178f4b4382cb78892cdd1bf1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/4a747b10178f4b4382cb78892cdd1bf1 2024-12-06T14:44:06,787 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into 4a747b10178f4b4382cb78892cdd1bf1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:06,787 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:06,787 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=13, startTime=1733496246369; duration=0sec 2024-12-06T14:44:06,787 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:06,787 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:44:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496306813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T14:44:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496306928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:06,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T14:44:06,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:06,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:06,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:06,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:06,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:07,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T14:44:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:07,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T14:44:07,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496307138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:07,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/63744c9018b84bd4b425cb5c651d351f 2024-12-06T14:44:07,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/de67bf31e1e24c5b8b934d90a4e064b8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/de67bf31e1e24c5b8b934d90a4e064b8 2024-12-06T14:44:07,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/de67bf31e1e24c5b8b934d90a4e064b8, entries=200, sequenceid=358, filesize=14.4 K 2024-12-06T14:44:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/cc5cdb82b234446bb1772117c9d8c104 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/cc5cdb82b234446bb1772117c9d8c104 2024-12-06T14:44:07,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/cc5cdb82b234446bb1772117c9d8c104, entries=150, sequenceid=358, filesize=12.0 K 2024-12-06T14:44:07,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/63744c9018b84bd4b425cb5c651d351f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/63744c9018b84bd4b425cb5c651d351f 2024-12-06T14:44:07,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/63744c9018b84bd4b425cb5c651d351f, entries=150, sequenceid=358, filesize=12.0 K 2024-12-06T14:44:07,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e75a56a757f742066bce3d3e372aff1b in 437ms, sequenceid=358, compaction requested=false 2024-12-06T14:44:07,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:07,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:07,237 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:07,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/c2f098c3966c469b9b91e4ec69f46068 is 50, key is test_row_0/A:col10/1733496246812/Put/seqid=0 2024-12-06T14:44:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742333_1509 (size=12301) 2024-12-06T14:44:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T14:44:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. as already flushing 2024-12-06T14:44:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496307555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:07,645 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/c2f098c3966c469b9b91e4ec69f46068 2024-12-06T14:44:07,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c6346fb9e0944c4c85ae61967f9464b7 is 50, key is test_row_0/B:col10/1733496246812/Put/seqid=0 2024-12-06T14:44:07,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742334_1510 (size=12301) 2024-12-06T14:44:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496307678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:07,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496307888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T14:44:08,059 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c6346fb9e0944c4c85ae61967f9464b7 2024-12-06T14:44:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/4036cbfa1a2443f1976a6bcd18cd2866 is 50, key is test_row_0/C:col10/1733496246812/Put/seqid=0 2024-12-06T14:44:08,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742335_1511 (size=12301) 2024-12-06T14:44:08,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1733496308198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 , the old value is region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=2 because the exception is null or not the one we care about 2024-12-06T14:44:08,467 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/4036cbfa1a2443f1976a6bcd18cd2866 2024-12-06T14:44:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/c2f098c3966c469b9b91e4ec69f46068 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/c2f098c3966c469b9b91e4ec69f46068 2024-12-06T14:44:08,472 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/c2f098c3966c469b9b91e4ec69f46068, entries=150, sequenceid=374, filesize=12.0 K 2024-12-06T14:44:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/c6346fb9e0944c4c85ae61967f9464b7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c6346fb9e0944c4c85ae61967f9464b7 2024-12-06T14:44:08,478 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c6346fb9e0944c4c85ae61967f9464b7, entries=150, sequenceid=374, filesize=12.0 K 2024-12-06T14:44:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/4036cbfa1a2443f1976a6bcd18cd2866 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/4036cbfa1a2443f1976a6bcd18cd2866 2024-12-06T14:44:08,482 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/4036cbfa1a2443f1976a6bcd18cd2866, entries=150, sequenceid=374, filesize=12.0 K 2024-12-06T14:44:08,482 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e75a56a757f742066bce3d3e372aff1b in 1245ms, sequenceid=374, compaction requested=true 2024-12-06T14:44:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-06T14:44:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-06T14:44:08,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-06T14:44:08,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-12-06T14:44:08,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7070 sec 2024-12-06T14:44:08,547 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T14:44:08,696 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-12-06T14:44:08,696 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:08,696 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,697 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:08,699 INFO [Thread-1924 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1924. 2024-12-06T14:44:08,699 DEBUG [Thread-1924 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:08,699 DEBUG [Thread-1924 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,699 DEBUG [Thread-1924 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:08,701 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-12-06T14:44:08,701 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:08,701 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,701 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:08,702 INFO [Thread-1922 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1922. 2024-12-06T14:44:08,702 DEBUG [Thread-1922 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:08,703 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,703 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:08,704 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-12-06T14:44:08,704 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:08,704 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,704 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:08,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:44:08,715 INFO [Thread-1917 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1917. 2024-12-06T14:44:08,715 DEBUG [Thread-1917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:08,715 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:08,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:08,715 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:08,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:08,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:08,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:08,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:08,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:08,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/290ca9e0772e4b5e959305d4c1791c98 is 50, key is test_row_0/A:col10/1733496247546/Put/seqid=0 2024-12-06T14:44:08,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742336_1512 (size=12301) 2024-12-06T14:44:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T14:44:08,938 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:09,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/290ca9e0772e4b5e959305d4c1791c98 2024-12-06T14:44:09,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/1a1dc4049c44463ebaddf01e6eb8af8f is 50, key is test_row_0/B:col10/1733496247546/Put/seqid=0 2024-12-06T14:44:09,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742337_1513 (size=12301) 2024-12-06T14:44:09,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/1a1dc4049c44463ebaddf01e6eb8af8f 2024-12-06T14:44:09,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/1cebf64d750340dbbaf7fb527f7edea4 is 50, key is test_row_0/C:col10/1733496247546/Put/seqid=0 2024-12-06T14:44:09,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742338_1514 (size=12301) 2024-12-06T14:44:09,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/1cebf64d750340dbbaf7fb527f7edea4 2024-12-06T14:44:09,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/290ca9e0772e4b5e959305d4c1791c98 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/290ca9e0772e4b5e959305d4c1791c98 2024-12-06T14:44:09,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/290ca9e0772e4b5e959305d4c1791c98, entries=150, sequenceid=392, filesize=12.0 K 2024-12-06T14:44:09,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/1a1dc4049c44463ebaddf01e6eb8af8f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/1a1dc4049c44463ebaddf01e6eb8af8f 2024-12-06T14:44:09,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/1a1dc4049c44463ebaddf01e6eb8af8f, entries=150, sequenceid=392, filesize=12.0 K 2024-12-06T14:44:09,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/1cebf64d750340dbbaf7fb527f7edea4 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/1cebf64d750340dbbaf7fb527f7edea4 2024-12-06T14:44:09,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/1cebf64d750340dbbaf7fb527f7edea4, entries=150, sequenceid=392, filesize=12.0 K 2024-12-06T14:44:09,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for e75a56a757f742066bce3d3e372aff1b in 1251ms, sequenceid=392, compaction requested=true 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:09,966 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e75a56a757f742066bce3d3e372aff1b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:09,966 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:09,967 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:09,967 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:09,967 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/A is initiating minor compaction (all files) 2024-12-06T14:44:09,967 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/B is initiating minor compaction (all files) 2024-12-06T14:44:09,967 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/A in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:09,967 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/B in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:09,968 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85d6672a1b73493dba0a3eaa0dc785b6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/de67bf31e1e24c5b8b934d90a4e064b8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/c2f098c3966c469b9b91e4ec69f46068, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/290ca9e0772e4b5e959305d4c1791c98] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=51.3 K 2024-12-06T14:44:09,968 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/4a747b10178f4b4382cb78892cdd1bf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/cc5cdb82b234446bb1772117c9d8c104, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c6346fb9e0944c4c85ae61967f9464b7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/1a1dc4049c44463ebaddf01e6eb8af8f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=48.9 K 2024-12-06T14:44:09,968 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d6672a1b73493dba0a3eaa0dc785b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496244890 2024-12-06T14:44:09,968 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a747b10178f4b4382cb78892cdd1bf1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496244890 2024-12-06T14:44:09,968 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting de67bf31e1e24c5b8b934d90a4e064b8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733496245573 2024-12-06T14:44:09,968 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5cdb82b234446bb1772117c9d8c104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733496245588 2024-12-06T14:44:09,969 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2f098c3966c469b9b91e4ec69f46068, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733496246777 2024-12-06T14:44:09,969 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c6346fb9e0944c4c85ae61967f9464b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733496246777 2024-12-06T14:44:09,969 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 290ca9e0772e4b5e959305d4c1791c98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496247539 2024-12-06T14:44:09,969 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1dc4049c44463ebaddf01e6eb8af8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496247539 2024-12-06T14:44:09,977 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:09,977 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:09,977 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/f322bcfe3dd44b1693d8d3f3d47f0f39 is 50, key is test_row_0/A:col10/1733496247546/Put/seqid=0 2024-12-06T14:44:09,977 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/7887d9dfbe4e494e9b6f9a081d25ab12 is 50, key is test_row_0/B:col10/1733496247546/Put/seqid=0 2024-12-06T14:44:09,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742339_1515 (size=13323) 2024-12-06T14:44:09,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742340_1516 (size=13323) 2024-12-06T14:44:10,391 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/7887d9dfbe4e494e9b6f9a081d25ab12 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/7887d9dfbe4e494e9b6f9a081d25ab12 2024-12-06T14:44:10,392 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/f322bcfe3dd44b1693d8d3f3d47f0f39 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/f322bcfe3dd44b1693d8d3f3d47f0f39 2024-12-06T14:44:10,394 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/B of e75a56a757f742066bce3d3e372aff1b into 7887d9dfbe4e494e9b6f9a081d25ab12(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:10,394 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:10,394 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/A of e75a56a757f742066bce3d3e372aff1b into f322bcfe3dd44b1693d8d3f3d47f0f39(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:10,394 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:10,394 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/B, priority=12, startTime=1733496249966; duration=0sec 2024-12-06T14:44:10,394 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/A, priority=12, startTime=1733496249966; duration=0sec 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:A 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:B 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:10,395 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): e75a56a757f742066bce3d3e372aff1b/C is initiating minor compaction (all files) 2024-12-06T14:44:10,395 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e75a56a757f742066bce3d3e372aff1b/C in TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:10,395 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/20f33f1e8c6c4cc6904fba1bc2a83cad, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/63744c9018b84bd4b425cb5c651d351f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/4036cbfa1a2443f1976a6bcd18cd2866, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/1cebf64d750340dbbaf7fb527f7edea4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp, totalSize=48.9 K 2024-12-06T14:44:10,396 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20f33f1e8c6c4cc6904fba1bc2a83cad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733496244890 2024-12-06T14:44:10,396 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63744c9018b84bd4b425cb5c651d351f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733496245588 2024-12-06T14:44:10,396 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4036cbfa1a2443f1976a6bcd18cd2866, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733496246777 2024-12-06T14:44:10,396 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cebf64d750340dbbaf7fb527f7edea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733496247539 2024-12-06T14:44:10,401 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e75a56a757f742066bce3d3e372aff1b#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:10,401 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/72cc906da4d3439eaf99fbd42ff95748 is 50, key is test_row_0/C:col10/1733496247546/Put/seqid=0 2024-12-06T14:44:10,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742341_1517 (size=13323) 2024-12-06T14:44:10,811 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/72cc906da4d3439eaf99fbd42ff95748 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/72cc906da4d3439eaf99fbd42ff95748 2024-12-06T14:44:10,818 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e75a56a757f742066bce3d3e372aff1b/C of e75a56a757f742066bce3d3e372aff1b into 72cc906da4d3439eaf99fbd42ff95748(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:10,818 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e75a56a757f742066bce3d3e372aff1b: 2024-12-06T14:44:10,818 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., storeName=e75a56a757f742066bce3d3e372aff1b/C, priority=12, startTime=1733496249966; duration=0sec 2024-12-06T14:44:10,818 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:10,818 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e75a56a757f742066bce3d3e372aff1b:C 2024-12-06T14:44:12,889 INFO [Thread-1913 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1913. 2024-12-06T14:44:12,889 DEBUG [Thread-1913 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:12,889 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,889 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:12,899 INFO [Thread-1915 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1915. 2024-12-06T14:44:12,899 DEBUG [Thread-1915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:12,899 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,899 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:12,959 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-12-06T14:44:12,959 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:12,959 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,960 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:12,969 INFO [Thread-1911 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1911. 2024-12-06T14:44:12,970 DEBUG [Thread-1911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:12,970 DEBUG [Thread-1911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,970 DEBUG [Thread-1911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 220 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1750 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5246 rows 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1740 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5219 rows 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1749 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5247 rows 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1764 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5292 rows 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1743 2024-12-06T14:44:12,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5222 rows 2024-12-06T14:44:12,970 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:44:12,970 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:44:12,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:12,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:12,971 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T14:44:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T14:44:12,974 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496252974"}]},"ts":"1733496252974"} 2024-12-06T14:44:12,976 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T14:44:12,976 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T14:44:12,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:44:12,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, UNASSIGN}] 2024-12-06T14:44:12,978 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, UNASSIGN 2024-12-06T14:44:12,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=e75a56a757f742066bce3d3e372aff1b, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:12,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, UNASSIGN because future has completed 2024-12-06T14:44:12,981 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:44:12,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:44:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T14:44:13,133 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing e75a56a757f742066bce3d3e372aff1b, disabling compactions & flushes 2024-12-06T14:44:13,133 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. after waiting 0 ms 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:13,133 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing e75a56a757f742066bce3d3e372aff1b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=A 2024-12-06T14:44:13,133 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:13,134 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=B 2024-12-06T14:44:13,134 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:13,134 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e75a56a757f742066bce3d3e372aff1b, store=C 2024-12-06T14:44:13,134 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:13,137 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/e86738cbbf864b5db1bdefca18cc7711 is 50, key is test_row_1/A:col10/1733496252968/Put/seqid=0 2024-12-06T14:44:13,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742342_1518 (size=9857) 2024-12-06T14:44:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T14:44:13,541 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/e86738cbbf864b5db1bdefca18cc7711 2024-12-06T14:44:13,551 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/363d768fa412482888c377909a499efb is 50, key is test_row_1/B:col10/1733496252968/Put/seqid=0 2024-12-06T14:44:13,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742343_1519 (size=9857) 2024-12-06T14:44:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T14:44:13,955 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/363d768fa412482888c377909a499efb 2024-12-06T14:44:13,963 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/d8bf91df9a73424b84919c5fbe8c1bef is 50, key is test_row_1/C:col10/1733496252968/Put/seqid=0 2024-12-06T14:44:13,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742344_1520 (size=9857) 2024-12-06T14:44:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T14:44:14,368 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/d8bf91df9a73424b84919c5fbe8c1bef 2024-12-06T14:44:14,378 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/A/e86738cbbf864b5db1bdefca18cc7711 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e86738cbbf864b5db1bdefca18cc7711 2024-12-06T14:44:14,382 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e86738cbbf864b5db1bdefca18cc7711, entries=100, sequenceid=402, filesize=9.6 K 2024-12-06T14:44:14,382 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/B/363d768fa412482888c377909a499efb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/363d768fa412482888c377909a499efb 2024-12-06T14:44:14,386 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/363d768fa412482888c377909a499efb, entries=100, sequenceid=402, filesize=9.6 K 2024-12-06T14:44:14,386 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/.tmp/C/d8bf91df9a73424b84919c5fbe8c1bef as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d8bf91df9a73424b84919c5fbe8c1bef 2024-12-06T14:44:14,390 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d8bf91df9a73424b84919c5fbe8c1bef, entries=100, sequenceid=402, filesize=9.6 K 2024-12-06T14:44:14,391 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e75a56a757f742066bce3d3e372aff1b in 1258ms, sequenceid=402, compaction requested=false 2024-12-06T14:44:14,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ebf87e7ce76a413ba73a80450a518798, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/8e746ba3ad9142f1bf243a874366ceb4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/fb3ed7368a46489183767815713cd60a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ed371c0123ef40cf848659518654b34e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/60906399466c4eb68ad7e1184fca09f3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85bd74fd5575484a876a5eaf2374c2ab, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/1c96c92ab07a4cd5b549410d3dca3399, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/12f946232aea41e69d5b9967ee197b5f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ba845d4ed7a34673abfb046ccc71f9ab, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/aeb71507df76471b86af13de76a4b10a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/db07ff67acdb4988ab05a491639c5491, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6a37ad0688bc4d3781e182e50dc99773, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6e2f00c4aa4d45cbbb947a757e1b08ce, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e60d8177fe1646a1858f4e5151148e11, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/20b25252a7a64d8fad663a86abfd6903, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/78e1565f6bdd40b1abbfc2d59a0cba7d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/868a19f4a6a749b780251d92b18afa79, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/7ffd16ca220743d19a14d585592f7a06, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/90b7f7ca275f49e1a6ac22d1dee5e9dd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ab982149335a4900816a509ada63937c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/99d063620f61455894233f84647fa02f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/591116f71f104a30a4f11cd9a4a4ba63, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/a21da74c1aa54fa68d09ad392d71550a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/93e37b0c2f0d403fbd4cba7070fc1e3f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/2c5f62288cfd4409bcb18348e09660ea, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/d73bf34412f34b83ac41d5840cf0e38a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85d6672a1b73493dba0a3eaa0dc785b6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/de67bf31e1e24c5b8b934d90a4e064b8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/c2f098c3966c469b9b91e4ec69f46068, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/290ca9e0772e4b5e959305d4c1791c98] to archive 2024-12-06T14:44:14,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:44:14,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ebf87e7ce76a413ba73a80450a518798 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ebf87e7ce76a413ba73a80450a518798 2024-12-06T14:44:14,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/8e746ba3ad9142f1bf243a874366ceb4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/8e746ba3ad9142f1bf243a874366ceb4 2024-12-06T14:44:14,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/fb3ed7368a46489183767815713cd60a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/fb3ed7368a46489183767815713cd60a 2024-12-06T14:44:14,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ed371c0123ef40cf848659518654b34e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ed371c0123ef40cf848659518654b34e 2024-12-06T14:44:14,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/60906399466c4eb68ad7e1184fca09f3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/60906399466c4eb68ad7e1184fca09f3 2024-12-06T14:44:14,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85bd74fd5575484a876a5eaf2374c2ab to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85bd74fd5575484a876a5eaf2374c2ab 2024-12-06T14:44:14,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/1c96c92ab07a4cd5b549410d3dca3399 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/1c96c92ab07a4cd5b549410d3dca3399 2024-12-06T14:44:14,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/12f946232aea41e69d5b9967ee197b5f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/12f946232aea41e69d5b9967ee197b5f 2024-12-06T14:44:14,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ba845d4ed7a34673abfb046ccc71f9ab to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ba845d4ed7a34673abfb046ccc71f9ab 2024-12-06T14:44:14,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/aeb71507df76471b86af13de76a4b10a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/aeb71507df76471b86af13de76a4b10a 2024-12-06T14:44:14,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/db07ff67acdb4988ab05a491639c5491 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/db07ff67acdb4988ab05a491639c5491 2024-12-06T14:44:14,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6a37ad0688bc4d3781e182e50dc99773 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6a37ad0688bc4d3781e182e50dc99773 2024-12-06T14:44:14,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6e2f00c4aa4d45cbbb947a757e1b08ce to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/6e2f00c4aa4d45cbbb947a757e1b08ce 2024-12-06T14:44:14,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e60d8177fe1646a1858f4e5151148e11 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e60d8177fe1646a1858f4e5151148e11 2024-12-06T14:44:14,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/20b25252a7a64d8fad663a86abfd6903 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/20b25252a7a64d8fad663a86abfd6903 2024-12-06T14:44:14,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/78e1565f6bdd40b1abbfc2d59a0cba7d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/78e1565f6bdd40b1abbfc2d59a0cba7d 2024-12-06T14:44:14,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/868a19f4a6a749b780251d92b18afa79 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/868a19f4a6a749b780251d92b18afa79 2024-12-06T14:44:14,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/7ffd16ca220743d19a14d585592f7a06 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/7ffd16ca220743d19a14d585592f7a06 2024-12-06T14:44:14,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/90b7f7ca275f49e1a6ac22d1dee5e9dd to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/90b7f7ca275f49e1a6ac22d1dee5e9dd 2024-12-06T14:44:14,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ab982149335a4900816a509ada63937c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/ab982149335a4900816a509ada63937c 2024-12-06T14:44:14,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/99d063620f61455894233f84647fa02f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/99d063620f61455894233f84647fa02f 2024-12-06T14:44:14,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/591116f71f104a30a4f11cd9a4a4ba63 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/591116f71f104a30a4f11cd9a4a4ba63 2024-12-06T14:44:14,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/a21da74c1aa54fa68d09ad392d71550a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/a21da74c1aa54fa68d09ad392d71550a 2024-12-06T14:44:14,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/93e37b0c2f0d403fbd4cba7070fc1e3f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/93e37b0c2f0d403fbd4cba7070fc1e3f 2024-12-06T14:44:14,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/2c5f62288cfd4409bcb18348e09660ea to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/2c5f62288cfd4409bcb18348e09660ea 2024-12-06T14:44:14,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/d73bf34412f34b83ac41d5840cf0e38a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/d73bf34412f34b83ac41d5840cf0e38a 2024-12-06T14:44:14,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85d6672a1b73493dba0a3eaa0dc785b6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/85d6672a1b73493dba0a3eaa0dc785b6 2024-12-06T14:44:14,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/de67bf31e1e24c5b8b934d90a4e064b8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/de67bf31e1e24c5b8b934d90a4e064b8 2024-12-06T14:44:14,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/c2f098c3966c469b9b91e4ec69f46068 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/c2f098c3966c469b9b91e4ec69f46068 2024-12-06T14:44:14,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/290ca9e0772e4b5e959305d4c1791c98 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/290ca9e0772e4b5e959305d4c1791c98 2024-12-06T14:44:14,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/043e4270e53d4277ae5f68f099206ee2, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/25197219242b4c688d7b0b6d67a961a8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/9b8a622c20cb42119adee60d490234d4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/17a06fda06524714bdd407111d225a21, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/b1317972445749ae84a5a8820381aab8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/fb62464619304ad99b3b6b9a041a7924, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/385b5c33d906459db3d405fab0208e53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0e655ee236b54a63b9dc16ae33b4cee1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3401e7314eb6446c943b4d4309048ba4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/d8ff0dc40b6949268115b7e2b624dd33, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/de681a6247f14fc5bec84f607a2b3f10, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/ab7673c6011b41e4a55469f1d0428a3b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3e44b5e45fc743969d1ff7864b832444, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/80296c7c0b7b42828d38ce41ea42761b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f67fca7802ea40c2b6f683680a050515, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c1fbd8ec0ecc432fa90426eca5493d6c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/413d19c2c3064bb0a8c9495be9ec456b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/107078857a06454a9cc583b31837a207, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3d9d397763144fc88f65c220eae8e35f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/110c4dff521b4dc3b9c32d0bda228c69, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f033751312514b91b5b225b644394e04, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f7a42d193ad64a38beb7652134a94cf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/46adf004f5cd445088548bed9e64cb95, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/091d268a05fb404bb4877212c9591b79, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0a9dbb9898ab4bb1bee918d05b9fb419, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/4a747b10178f4b4382cb78892cdd1bf1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c2e733fc5a2342eba3e4bf4bd3a43dd6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/cc5cdb82b234446bb1772117c9d8c104, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c6346fb9e0944c4c85ae61967f9464b7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/1a1dc4049c44463ebaddf01e6eb8af8f] to archive 2024-12-06T14:44:14,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:44:14,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/043e4270e53d4277ae5f68f099206ee2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/043e4270e53d4277ae5f68f099206ee2 2024-12-06T14:44:14,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/25197219242b4c688d7b0b6d67a961a8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/25197219242b4c688d7b0b6d67a961a8 2024-12-06T14:44:14,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/9b8a622c20cb42119adee60d490234d4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/9b8a622c20cb42119adee60d490234d4 2024-12-06T14:44:14,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/17a06fda06524714bdd407111d225a21 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/17a06fda06524714bdd407111d225a21 2024-12-06T14:44:14,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/b1317972445749ae84a5a8820381aab8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/b1317972445749ae84a5a8820381aab8 2024-12-06T14:44:14,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/fb62464619304ad99b3b6b9a041a7924 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/fb62464619304ad99b3b6b9a041a7924 2024-12-06T14:44:14,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/385b5c33d906459db3d405fab0208e53 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/385b5c33d906459db3d405fab0208e53 2024-12-06T14:44:14,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0e655ee236b54a63b9dc16ae33b4cee1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0e655ee236b54a63b9dc16ae33b4cee1 2024-12-06T14:44:14,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3401e7314eb6446c943b4d4309048ba4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3401e7314eb6446c943b4d4309048ba4 2024-12-06T14:44:14,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/d8ff0dc40b6949268115b7e2b624dd33 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/d8ff0dc40b6949268115b7e2b624dd33 2024-12-06T14:44:14,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/de681a6247f14fc5bec84f607a2b3f10 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/de681a6247f14fc5bec84f607a2b3f10 2024-12-06T14:44:14,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/ab7673c6011b41e4a55469f1d0428a3b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/ab7673c6011b41e4a55469f1d0428a3b 2024-12-06T14:44:14,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3e44b5e45fc743969d1ff7864b832444 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3e44b5e45fc743969d1ff7864b832444 2024-12-06T14:44:14,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/80296c7c0b7b42828d38ce41ea42761b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/80296c7c0b7b42828d38ce41ea42761b 2024-12-06T14:44:14,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f67fca7802ea40c2b6f683680a050515 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f67fca7802ea40c2b6f683680a050515 2024-12-06T14:44:14,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c1fbd8ec0ecc432fa90426eca5493d6c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c1fbd8ec0ecc432fa90426eca5493d6c 2024-12-06T14:44:14,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/413d19c2c3064bb0a8c9495be9ec456b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/413d19c2c3064bb0a8c9495be9ec456b 2024-12-06T14:44:14,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/107078857a06454a9cc583b31837a207 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/107078857a06454a9cc583b31837a207 2024-12-06T14:44:14,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3d9d397763144fc88f65c220eae8e35f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/3d9d397763144fc88f65c220eae8e35f 2024-12-06T14:44:14,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/110c4dff521b4dc3b9c32d0bda228c69 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/110c4dff521b4dc3b9c32d0bda228c69 2024-12-06T14:44:14,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f033751312514b91b5b225b644394e04 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f033751312514b91b5b225b644394e04 2024-12-06T14:44:14,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f7a42d193ad64a38beb7652134a94cf1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/f7a42d193ad64a38beb7652134a94cf1 2024-12-06T14:44:14,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/46adf004f5cd445088548bed9e64cb95 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/46adf004f5cd445088548bed9e64cb95 2024-12-06T14:44:14,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/091d268a05fb404bb4877212c9591b79 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/091d268a05fb404bb4877212c9591b79 2024-12-06T14:44:14,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0a9dbb9898ab4bb1bee918d05b9fb419 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/0a9dbb9898ab4bb1bee918d05b9fb419 2024-12-06T14:44:14,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/4a747b10178f4b4382cb78892cdd1bf1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/4a747b10178f4b4382cb78892cdd1bf1 2024-12-06T14:44:14,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c2e733fc5a2342eba3e4bf4bd3a43dd6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c2e733fc5a2342eba3e4bf4bd3a43dd6 2024-12-06T14:44:14,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/cc5cdb82b234446bb1772117c9d8c104 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/cc5cdb82b234446bb1772117c9d8c104 2024-12-06T14:44:14,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c6346fb9e0944c4c85ae61967f9464b7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/c6346fb9e0944c4c85ae61967f9464b7 2024-12-06T14:44:14,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/1a1dc4049c44463ebaddf01e6eb8af8f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/1a1dc4049c44463ebaddf01e6eb8af8f 2024-12-06T14:44:14,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/5f2937eed8064b70a058bffa529bff02, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/24b733f0758743f48f72ab11147fa317, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/cb42bc18945741749d882c420ae68719, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ac51fdfc6b2f45f086a858b2bb4be6cd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/f675771c8804457490f2885b01c80bbf, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/93cebf93bce34934baade8e8d67645b4, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/c44d9f3432f140cfa0c03a7c64aa46b3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e01b3bdb97e4ad3b54f2b45180f519f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/81148de3e6d0479fa3f498123f972b77, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6cfb820348a04db2a00bbeeaddcdf6a0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6f301a2d0a1492d87f6b89ecccef09c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/de64af1f69334eb9b9b25869b06b1958, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/82bffd63ad5e4bc3b0120ff38a292b07, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ecc906c288304be9bb249d440959a52d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/7f1d6e98ed43420797b7310f84888601, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/96fe59656d004c30959a00ef208f101e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/967365ad34cb4526ba3417dbe9ffd8aa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/e54aa638285641d8b35b0f4924a45ee1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/02ac9ac564784d0ca10f2b2735db7a0b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e9b7ffbcd1d4604a3f68df128be97a1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/caa0d87faccf4343a2fb0859ec22253e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d3cac2befb3e4986855143bea706c28b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/474e4193803f4d059ec856c8e2c3bf37, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/581641b1b8124560a2482f0d0887f7fa, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6c6ea83bb5a4531a1e367b7ff950b74, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/20f33f1e8c6c4cc6904fba1bc2a83cad, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/12354ea2a3dd4555afd9d81c40b97e1d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/63744c9018b84bd4b425cb5c651d351f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/4036cbfa1a2443f1976a6bcd18cd2866, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/1cebf64d750340dbbaf7fb527f7edea4] to archive 2024-12-06T14:44:14,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:44:14,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/5f2937eed8064b70a058bffa529bff02 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/5f2937eed8064b70a058bffa529bff02 2024-12-06T14:44:14,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/24b733f0758743f48f72ab11147fa317 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/24b733f0758743f48f72ab11147fa317 2024-12-06T14:44:14,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/cb42bc18945741749d882c420ae68719 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/cb42bc18945741749d882c420ae68719 2024-12-06T14:44:14,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ac51fdfc6b2f45f086a858b2bb4be6cd to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ac51fdfc6b2f45f086a858b2bb4be6cd 2024-12-06T14:44:14,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/f675771c8804457490f2885b01c80bbf to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/f675771c8804457490f2885b01c80bbf 2024-12-06T14:44:14,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/93cebf93bce34934baade8e8d67645b4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/93cebf93bce34934baade8e8d67645b4 2024-12-06T14:44:14,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/c44d9f3432f140cfa0c03a7c64aa46b3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/c44d9f3432f140cfa0c03a7c64aa46b3 2024-12-06T14:44:14,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e01b3bdb97e4ad3b54f2b45180f519f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e01b3bdb97e4ad3b54f2b45180f519f 2024-12-06T14:44:14,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/81148de3e6d0479fa3f498123f972b77 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/81148de3e6d0479fa3f498123f972b77 2024-12-06T14:44:14,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6cfb820348a04db2a00bbeeaddcdf6a0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6cfb820348a04db2a00bbeeaddcdf6a0 2024-12-06T14:44:14,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6f301a2d0a1492d87f6b89ecccef09c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6f301a2d0a1492d87f6b89ecccef09c 2024-12-06T14:44:14,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/de64af1f69334eb9b9b25869b06b1958 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/de64af1f69334eb9b9b25869b06b1958 2024-12-06T14:44:14,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/82bffd63ad5e4bc3b0120ff38a292b07 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/82bffd63ad5e4bc3b0120ff38a292b07 2024-12-06T14:44:14,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ecc906c288304be9bb249d440959a52d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/ecc906c288304be9bb249d440959a52d 2024-12-06T14:44:14,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/7f1d6e98ed43420797b7310f84888601 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/7f1d6e98ed43420797b7310f84888601 2024-12-06T14:44:14,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/96fe59656d004c30959a00ef208f101e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/96fe59656d004c30959a00ef208f101e 2024-12-06T14:44:14,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/967365ad34cb4526ba3417dbe9ffd8aa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/967365ad34cb4526ba3417dbe9ffd8aa 2024-12-06T14:44:14,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/e54aa638285641d8b35b0f4924a45ee1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/e54aa638285641d8b35b0f4924a45ee1 2024-12-06T14:44:14,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/02ac9ac564784d0ca10f2b2735db7a0b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/02ac9ac564784d0ca10f2b2735db7a0b 2024-12-06T14:44:14,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e9b7ffbcd1d4604a3f68df128be97a1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/6e9b7ffbcd1d4604a3f68df128be97a1 2024-12-06T14:44:14,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/caa0d87faccf4343a2fb0859ec22253e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/caa0d87faccf4343a2fb0859ec22253e 2024-12-06T14:44:14,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d3cac2befb3e4986855143bea706c28b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d3cac2befb3e4986855143bea706c28b 2024-12-06T14:44:14,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/474e4193803f4d059ec856c8e2c3bf37 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/474e4193803f4d059ec856c8e2c3bf37 2024-12-06T14:44:14,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/581641b1b8124560a2482f0d0887f7fa to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/581641b1b8124560a2482f0d0887f7fa 2024-12-06T14:44:14,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6c6ea83bb5a4531a1e367b7ff950b74 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/a6c6ea83bb5a4531a1e367b7ff950b74 2024-12-06T14:44:14,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/20f33f1e8c6c4cc6904fba1bc2a83cad to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/20f33f1e8c6c4cc6904fba1bc2a83cad 2024-12-06T14:44:14,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/12354ea2a3dd4555afd9d81c40b97e1d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/12354ea2a3dd4555afd9d81c40b97e1d 2024-12-06T14:44:14,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/63744c9018b84bd4b425cb5c651d351f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/63744c9018b84bd4b425cb5c651d351f 2024-12-06T14:44:14,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/4036cbfa1a2443f1976a6bcd18cd2866 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/4036cbfa1a2443f1976a6bcd18cd2866 2024-12-06T14:44:14,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/1cebf64d750340dbbaf7fb527f7edea4 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/1cebf64d750340dbbaf7fb527f7edea4 2024-12-06T14:44:14,456 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/recovered.edits/405.seqid, newMaxSeqId=405, maxSeqId=1 2024-12-06T14:44:14,456 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. 2024-12-06T14:44:14,456 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for e75a56a757f742066bce3d3e372aff1b: Waiting for close lock at 1733496253133Running coprocessor pre-close hooks at 1733496253133Disabling compacts and flushes for region at 1733496253133Disabling writes for close at 1733496253133Obtaining lock to block concurrent updates at 1733496253133Preparing flush snapshotting stores in e75a56a757f742066bce3d3e372aff1b at 1733496253133Finished memstore snapshotting TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733496253134 (+1 ms)Flushing stores of TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b. at 1733496253134Flushing e75a56a757f742066bce3d3e372aff1b/A: creating writer at 1733496253134Flushing e75a56a757f742066bce3d3e372aff1b/A: appending metadata at 1733496253136 (+2 ms)Flushing e75a56a757f742066bce3d3e372aff1b/A: closing flushed file at 1733496253136Flushing e75a56a757f742066bce3d3e372aff1b/B: creating writer at 1733496253546 (+410 ms)Flushing e75a56a757f742066bce3d3e372aff1b/B: appending metadata at 1733496253550 (+4 ms)Flushing e75a56a757f742066bce3d3e372aff1b/B: closing flushed file at 1733496253550Flushing e75a56a757f742066bce3d3e372aff1b/C: creating writer at 1733496253959 (+409 ms)Flushing e75a56a757f742066bce3d3e372aff1b/C: appending metadata at 1733496253962 (+3 ms)Flushing e75a56a757f742066bce3d3e372aff1b/C: closing flushed file at 1733496253962Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46966391: reopening flushed file at 1733496254377 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@494f1cb7: reopening flushed file at 1733496254382 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a07868a: reopening flushed file at 1733496254386 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e75a56a757f742066bce3d3e372aff1b in 1258ms, sequenceid=402, compaction requested=false at 1733496254391 (+5 ms)Writing region close event to WAL at 1733496254454 (+63 ms)Running coprocessor post-close hooks at 1733496254456 (+2 ms)Closed at 1733496254456 2024-12-06T14:44:14,458 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:14,458 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=e75a56a757f742066bce3d3e372aff1b, regionState=CLOSED 2024-12-06T14:44:14,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:44:14,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-06T14:44:14,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure e75a56a757f742066bce3d3e372aff1b, server=bb7ea3e46d9c,35831,1733496100253 in 1.4790 sec 2024-12-06T14:44:14,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-06T14:44:14,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e75a56a757f742066bce3d3e372aff1b, UNASSIGN in 1.4840 sec 2024-12-06T14:44:14,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-06T14:44:14,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4880 sec 2024-12-06T14:44:14,467 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496254467"}]},"ts":"1733496254467"} 2024-12-06T14:44:14,468 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T14:44:14,468 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T14:44:14,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4980 sec 2024-12-06T14:44:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T14:44:15,119 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:15,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T14:44:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,125 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T14:44:15,127 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,129 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:15,132 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/recovered.edits] 2024-12-06T14:44:15,134 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e86738cbbf864b5db1bdefca18cc7711 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/e86738cbbf864b5db1bdefca18cc7711 2024-12-06T14:44:15,135 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/f322bcfe3dd44b1693d8d3f3d47f0f39 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/A/f322bcfe3dd44b1693d8d3f3d47f0f39 2024-12-06T14:44:15,137 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/363d768fa412482888c377909a499efb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/363d768fa412482888c377909a499efb 2024-12-06T14:44:15,137 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/7887d9dfbe4e494e9b6f9a081d25ab12 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/B/7887d9dfbe4e494e9b6f9a081d25ab12 2024-12-06T14:44:15,138 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/72cc906da4d3439eaf99fbd42ff95748 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/72cc906da4d3439eaf99fbd42ff95748 2024-12-06T14:44:15,139 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d8bf91df9a73424b84919c5fbe8c1bef to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/C/d8bf91df9a73424b84919c5fbe8c1bef 2024-12-06T14:44:15,140 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/recovered.edits/405.seqid to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b/recovered.edits/405.seqid 2024-12-06T14:44:15,140 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/e75a56a757f742066bce3d3e372aff1b 2024-12-06T14:44:15,141 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T14:44:15,142 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,143 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T14:44:15,145 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T14:44:15,145 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,145 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T14:44:15,146 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733496255145"}]},"ts":"9223372036854775807"} 2024-12-06T14:44:15,147 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T14:44:15,147 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e75a56a757f742066bce3d3e372aff1b, NAME => 'TestAcidGuarantees,,1733496226493.e75a56a757f742066bce3d3e372aff1b.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T14:44:15,147 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T14:44:15,147 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733496255147"}]},"ts":"9223372036854775807"} 2024-12-06T14:44:15,148 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T14:44:15,149 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-12-06T14:44:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T14:44:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T14:44:15,244 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:15,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=554 (was 577), ProcessCount=11 (was 11), AvailableMemoryMB=6682 (was 6842) 2024-12-06T14:44:15,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=554, ProcessCount=11, AvailableMemoryMB=6682 2024-12-06T14:44:15,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:44:15,266 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:44:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:15,268 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:44:15,268 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:15,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-06T14:44:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-06T14:44:15,269 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:44:15,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742345_1521 (size=1061) 2024-12-06T14:44:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-06T14:44:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-06T14:44:15,680 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5 2024-12-06T14:44:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742346_1522 (size=53) 2024-12-06T14:44:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-06T14:44:16,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:44:16,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3623e1fd493fe297ada6ccd9ac6123ae, disabling compactions & flushes 2024-12-06T14:44:16,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. after waiting 0 ms 2024-12-06T14:44:16,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3623e1fd493fe297ada6ccd9ac6123ae: Waiting for close lock at 1733496256090Disabling compacts and flushes for region at 1733496256090Disabling writes for close at 1733496256091 (+1 ms)Writing region close event to WAL at 1733496256091Closed at 1733496256091 2024-12-06T14:44:16,094 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:44:16,094 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733496256094"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496256094"}]},"ts":"1733496256094"} 2024-12-06T14:44:16,098 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:44:16,100 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:44:16,101 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496256100"}]},"ts":"1733496256100"} 2024-12-06T14:44:16,103 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T14:44:16,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, ASSIGN}] 2024-12-06T14:44:16,104 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, ASSIGN 2024-12-06T14:44:16,105 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=false 2024-12-06T14:44:16,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:16,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, ASSIGN because future has completed 2024-12-06T14:44:16,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:44:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-06T14:44:16,422 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,422 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:44:16,423 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,423 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:44:16,423 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,423 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,426 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,429 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:44:16,430 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3623e1fd493fe297ada6ccd9ac6123ae columnFamilyName A 2024-12-06T14:44:16,430 DEBUG [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:16,431 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(327): Store=3623e1fd493fe297ada6ccd9ac6123ae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:44:16,432 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,433 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:44:16,434 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3623e1fd493fe297ada6ccd9ac6123ae columnFamilyName B 2024-12-06T14:44:16,434 DEBUG [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:16,434 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(327): Store=3623e1fd493fe297ada6ccd9ac6123ae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:44:16,434 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,435 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:44:16,435 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3623e1fd493fe297ada6ccd9ac6123ae columnFamilyName C 2024-12-06T14:44:16,435 DEBUG [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:16,436 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(327): Store=3623e1fd493fe297ada6ccd9ac6123ae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:44:16,436 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,436 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,437 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,437 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,438 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,438 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,439 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:44:16,440 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,441 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:44:16,442 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 3623e1fd493fe297ada6ccd9ac6123ae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74338437, jitterRate=0.10772903263568878}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:44:16,442 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:16,443 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 3623e1fd493fe297ada6ccd9ac6123ae: Running coprocessor pre-open hook at 1733496256423Writing region info on filesystem at 1733496256423Initializing all the Stores at 1733496256425 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496256425Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496256426 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496256426Cleaning up temporary data from old regions at 1733496256438 (+12 ms)Running coprocessor post-open hooks at 1733496256442 (+4 ms)Region opened successfully at 1733496256442 2024-12-06T14:44:16,443 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., pid=140, masterSystemTime=1733496256415 2024-12-06T14:44:16,445 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,445 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:16,446 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:16,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:44:16,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-06T14:44:16,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 in 188 msec 2024-12-06T14:44:16,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-06T14:44:16,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, ASSIGN in 348 msec 2024-12-06T14:44:16,454 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:44:16,454 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496256454"}]},"ts":"1733496256454"} 2024-12-06T14:44:16,456 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T14:44:16,457 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:44:16,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1910 sec 2024-12-06T14:44:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-06T14:44:17,430 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:17,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:17,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:17,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:17,435 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652d66b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:17,438 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc4e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:17,439 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:17,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:17,442 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:44:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:44:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@177de511 2024-12-06T14:44:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:44:17,444 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:44:17,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T14:44:17,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:44:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:17,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742347_1523 (size=1097) 2024-12-06T14:44:17,859 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-06T14:44:17,860 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-06T14:44:17,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:44:17,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, REOPEN/MOVE}] 2024-12-06T14:44:17,868 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, REOPEN/MOVE 2024-12-06T14:44:17,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:17,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, REOPEN/MOVE because future has completed 2024-12-06T14:44:17,870 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:44:17,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:44:18,023 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,024 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:44:18,024 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 3623e1fd493fe297ada6ccd9ac6123ae, disabling compactions & flushes 2024-12-06T14:44:18,024 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,024 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,024 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. after waiting 0 ms 2024-12-06T14:44:18,024 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,031 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T14:44:18,032 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,032 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 3623e1fd493fe297ada6ccd9ac6123ae: Waiting for close lock at 1733496258024Running coprocessor pre-close hooks at 1733496258024Disabling compacts and flushes for region at 1733496258024Disabling writes for close at 1733496258024Writing region close event to WAL at 1733496258025 (+1 ms)Running coprocessor post-close hooks at 1733496258032 (+7 ms)Closed at 1733496258032 2024-12-06T14:44:18,032 WARN [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 3623e1fd493fe297ada6ccd9ac6123ae to self. 2024-12-06T14:44:18,035 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=CLOSED 2024-12-06T14:44:18,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:44:18,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-06T14:44:18,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 in 169 msec 2024-12-06T14:44:18,043 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, REOPEN/MOVE; state=CLOSED, location=bb7ea3e46d9c,35831,1733496100253; forceNewPlan=false, retain=true 2024-12-06T14:44:18,194 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=OPENING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, REOPEN/MOVE because future has completed 2024-12-06T14:44:18,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:44:18,355 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,356 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:44:18,356 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,356 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:44:18,356 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,357 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,359 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,360 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:44:18,360 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3623e1fd493fe297ada6ccd9ac6123ae columnFamilyName A 2024-12-06T14:44:18,362 DEBUG [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:18,363 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(327): Store=3623e1fd493fe297ada6ccd9ac6123ae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:44:18,364 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,365 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:44:18,365 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3623e1fd493fe297ada6ccd9ac6123ae columnFamilyName B 2024-12-06T14:44:18,365 DEBUG [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:18,366 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(327): Store=3623e1fd493fe297ada6ccd9ac6123ae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:44:18,366 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,366 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T14:44:18,367 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3623e1fd493fe297ada6ccd9ac6123ae columnFamilyName C 2024-12-06T14:44:18,367 DEBUG [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:18,367 INFO [StoreOpener-3623e1fd493fe297ada6ccd9ac6123ae-1 {}] regionserver.HStore(327): Store=3623e1fd493fe297ada6ccd9ac6123ae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:44:18,368 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,368 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,368 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,369 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,370 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,370 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,370 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T14:44:18,371 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,371 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 3623e1fd493fe297ada6ccd9ac6123ae; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70828266, jitterRate=0.05542340874671936}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T14:44:18,371 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,372 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 3623e1fd493fe297ada6ccd9ac6123ae: Running coprocessor pre-open hook at 1733496258357Writing region info on filesystem at 1733496258357Initializing all the Stores at 1733496258358 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496258358Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496258358Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496258358Cleaning up temporary data from old regions at 1733496258370 (+12 ms)Running coprocessor post-open hooks at 1733496258371 (+1 ms)Region opened successfully at 1733496258372 (+1 ms) 2024-12-06T14:44:18,372 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., pid=145, masterSystemTime=1733496258351 2024-12-06T14:44:18,374 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,374 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,374 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=OPEN, openSeqNum=5, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:44:18,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-06T14:44:18,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 in 179 msec 2024-12-06T14:44:18,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-06T14:44:18,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, REOPEN/MOVE in 510 msec 2024-12-06T14:44:18,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-06T14:44:18,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-12-06T14:44:18,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-12-06T14:44:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-06T14:44:18,382 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:18,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22842932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b6f650, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5829ff9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b1e75a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,388 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190cc578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,390 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aca4047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,391 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10815b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,393 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4226caf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,394 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ccae56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,396 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c95240e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,398 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d45a196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682fbbc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b718f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,404 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190b125f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,405 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c00c9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,407 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec9f762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,409 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a81a785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,411 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75bc824f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,413 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8c4dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,38707,-1 for getting cluster id 2024-12-06T14:44:18,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:44:18,415 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '346f2f90-4d95-4e78-9b2b-95d06fb591bd' 2024-12-06T14:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "346f2f90-4d95-4e78-9b2b-95d06fb591bd" 2024-12-06T14:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dea291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,38707,-1] 2024-12-06T14:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:18,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:44:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0bba69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:44:18,417 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,417 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [Thread-2349 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [Thread-2355 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [Thread-2351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,418 DEBUG [Thread-2353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,419 DEBUG [Thread-2357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,419 DEBUG [Thread-2344 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,419 DEBUG [Thread-2344 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,420 DEBUG [Thread-2338 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:44:18,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:18,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=-1] 2024-12-06T14:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:44:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-06T14:44:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,423 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:18,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:44:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,425 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5] 2024-12-06T14:44:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T14:44:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:18,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496258435, max timestamp=1733496258444Num uniques 859; 2024-12-06T14:44:18,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496258435, max timestamp=1733496258444Num uniques 859; 2024-12-06T14:44:18,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:44:18,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496258435, max timestamp=1733496258444Num uniques 859; 2024-12-06T14:44:18,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:44:18,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:44:18,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120681570d25c0be4ea6bc06cd17072bcf39_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496258426/Put/seqid=0 2024-12-06T14:44:18,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742348_1524 (size=12154) 2024-12-06T14:44:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496318455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496318456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496318457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496318458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496318459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496318568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496318568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496318568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496318568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496318569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:18,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:18,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496318778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496318778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496318778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496318778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496318778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:18,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:18,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120681570d25c0be4ea6bc06cd17072bcf39_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120681570d25c0be4ea6bc06cd17072bcf39_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:18,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/3b41905d46d14634937ee686122bc9f3, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:18,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/3b41905d46d14634937ee686122bc9f3 is 175, key is test_row_0/A:col10/1733496258426/Put/seqid=0 2024-12-06T14:44:18,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742349_1525 (size=30955) 2024-12-06T14:44:18,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:18,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496319088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496319088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496319088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496319088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496319088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,264 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/3b41905d46d14634937ee686122bc9f3 2024-12-06T14:44:19,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/7a54ed0b6a734a82bb5bd43cbb687890 is 50, key is test_row_0/B:col10/1733496258426/Put/seqid=0 2024-12-06T14:44:19,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742350_1526 (size=12001) 2024-12-06T14:44:19,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496319598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496319598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496319598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496319598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496319599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:19,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/7a54ed0b6a734a82bb5bd43cbb687890 2024-12-06T14:44:19,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cb7181e6e821469b99396a3f15813ae9 is 50, key is test_row_0/C:col10/1733496258426/Put/seqid=0 2024-12-06T14:44:19,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742351_1527 (size=12001) 2024-12-06T14:44:19,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:19,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:19,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:44:20,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:20,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:20,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:20,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:20,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:20,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cb7181e6e821469b99396a3f15813ae9 2024-12-06T14:44:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/3b41905d46d14634937ee686122bc9f3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3 2024-12-06T14:44:20,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3, entries=150, sequenceid=14, filesize=30.2 K 2024-12-06T14:44:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/7a54ed0b6a734a82bb5bd43cbb687890 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a54ed0b6a734a82bb5bd43cbb687890 2024-12-06T14:44:20,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a54ed0b6a734a82bb5bd43cbb687890, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:44:20,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cb7181e6e821469b99396a3f15813ae9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cb7181e6e821469b99396a3f15813ae9 2024-12-06T14:44:20,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cb7181e6e821469b99396a3f15813ae9, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T14:44:20,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3623e1fd493fe297ada6ccd9ac6123ae in 1775ms, sequenceid=14, compaction requested=false 2024-12-06T14:44:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:20,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T14:44:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:20,263 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-06T14:44:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063f2453764abb479ba5bb9d78408bdfdf_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496258448/Put/seqid=0 2024-12-06T14:44:20,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742352_1528 (size=12154) 2024-12-06T14:44:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,318 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063f2453764abb479ba5bb9d78408bdfdf_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063f2453764abb479ba5bb9d78408bdfdf_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:20,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2bac29a042964f2c9c884b29c1d15e9f, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2bac29a042964f2c9c884b29c1d15e9f is 175, key is test_row_0/A:col10/1733496258448/Put/seqid=0 2024-12-06T14:44:20,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742353_1529 (size=30955) 2024-12-06T14:44:20,364 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2bac29a042964f2c9c884b29c1d15e9f 2024-12-06T14:44:20,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/c5d52ee527094986808a8d2881d712f6 is 50, key is test_row_0/B:col10/1733496258448/Put/seqid=0 2024-12-06T14:44:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742354_1530 (size=12001) 2024-12-06T14:44:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:20,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496320618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496320618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496320618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496320619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,624 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,625 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496320622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496320728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496320728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496320738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:20,808 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/c5d52ee527094986808a8d2881d712f6 2024-12-06T14:44:20,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/5dab2d7e162441c5950f4821d5a0ee23 is 50, key is test_row_0/C:col10/1733496258448/Put/seqid=0 2024-12-06T14:44:20,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742355_1531 (size=12001) 2024-12-06T14:44:20,847 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/5dab2d7e162441c5950f4821d5a0ee23 2024-12-06T14:44:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2bac29a042964f2c9c884b29c1d15e9f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f 2024-12-06T14:44:20,857 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f, entries=150, sequenceid=50, filesize=30.2 K 2024-12-06T14:44:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/c5d52ee527094986808a8d2881d712f6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c5d52ee527094986808a8d2881d712f6 2024-12-06T14:44:20,863 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c5d52ee527094986808a8d2881d712f6, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T14:44:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/5dab2d7e162441c5950f4821d5a0ee23 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5dab2d7e162441c5950f4821d5a0ee23 2024-12-06T14:44:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,868 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5dab2d7e162441c5950f4821d5a0ee23, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T14:44:20,869 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3623e1fd493fe297ada6ccd9ac6123ae in 606ms, sequenceid=50, compaction requested=false 2024-12-06T14:44:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-06T14:44:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-06T14:44:20,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-06T14:44:20,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4460 sec 2024-12-06T14:44:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.4560 sec 2024-12-06T14:44:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:20,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:44:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a6d025e96b7746b8b414e4312a67d508_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742356_1532 (size=14594) 2024-12-06T14:44:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496320998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496320998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496320999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496321318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496321319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496321322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,353 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:21,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a6d025e96b7746b8b414e4312a67d508_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a6d025e96b7746b8b414e4312a67d508_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:21,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/394bd7d77e684d94a4bbc08d3782ea90, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:21,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/394bd7d77e684d94a4bbc08d3782ea90 is 175, key is test_row_0/A:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:21,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742357_1533 (size=39545) 2024-12-06T14:44:21,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/394bd7d77e684d94a4bbc08d3782ea90 2024-12-06T14:44:21,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9b583a18987a4cf0b8d1e8c2eddd8abb is 50, key is test_row_0/B:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:21,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742358_1534 (size=9657) 2024-12-06T14:44:21,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9b583a18987a4cf0b8d1e8c2eddd8abb 2024-12-06T14:44:21,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/5c13cf4c740d4ffd8b9e00b0abfdeb53 is 50, key is test_row_0/C:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:21,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742359_1535 (size=9657) 2024-12-06T14:44:21,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/5c13cf4c740d4ffd8b9e00b0abfdeb53 2024-12-06T14:44:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/394bd7d77e684d94a4bbc08d3782ea90 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90 2024-12-06T14:44:21,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90, entries=200, sequenceid=62, filesize=38.6 K 2024-12-06T14:44:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9b583a18987a4cf0b8d1e8c2eddd8abb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9b583a18987a4cf0b8d1e8c2eddd8abb 2024-12-06T14:44:21,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9b583a18987a4cf0b8d1e8c2eddd8abb, entries=100, sequenceid=62, filesize=9.4 K 2024-12-06T14:44:21,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/5c13cf4c740d4ffd8b9e00b0abfdeb53 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5c13cf4c740d4ffd8b9e00b0abfdeb53 2024-12-06T14:44:21,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5c13cf4c740d4ffd8b9e00b0abfdeb53, entries=100, sequenceid=62, filesize=9.4 K 2024-12-06T14:44:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3623e1fd493fe297ada6ccd9ac6123ae in 610ms, sequenceid=62, compaction requested=true 2024-12-06T14:44:21,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:21,552 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:21,552 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:21,552 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:21,553 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:21,553 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a54ed0b6a734a82bb5bd43cbb687890, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c5d52ee527094986808a8d2881d712f6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9b583a18987a4cf0b8d1e8c2eddd8abb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=32.9 K 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:21,553 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:21,553 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=99.1 K 2024-12-06T14:44:21,553 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90] 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a54ed0b6a734a82bb5bd43cbb687890, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496258426 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b41905d46d14634937ee686122bc9f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496258426 2024-12-06T14:44:21,553 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c5d52ee527094986808a8d2881d712f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733496258448 2024-12-06T14:44:21,554 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bac29a042964f2c9c884b29c1d15e9f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733496258448 2024-12-06T14:44:21,554 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b583a18987a4cf0b8d1e8c2eddd8abb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733496260616 2024-12-06T14:44:21,554 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 394bd7d77e684d94a4bbc08d3782ea90, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733496260616 2024-12-06T14:44:21,565 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:21,592 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:21,593 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9828366e7dd24795a1e4942790f0edd8 is 50, key is test_row_0/B:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:21,594 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206c6ed221cb31f4ea6a26488bcb4a72c42_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:21,596 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206c6ed221cb31f4ea6a26488bcb4a72c42_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:21,596 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c6ed221cb31f4ea6a26488bcb4a72c42_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742360_1536 (size=4469) 2024-12-06T14:44:21,643 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#456 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:21,646 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/0f6e285f40d54b43b3e2835a95bde772 is 175, key is test_row_0/A:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:21,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496260987, max timestamp=1733496261648Num uniques 859; 2024-12-06T14:44:21,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:44:21,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496260987, max timestamp=1733496261648Num uniques 859; 2024-12-06T14:44:21,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:44:21,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733496260987, max timestamp=1733496261648Num uniques 859; 2024-12-06T14:44:21,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T14:44:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742361_1537 (size=12104) 2024-12-06T14:44:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:21,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-12-06T14:44:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:21,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742362_1538 (size=31165) 2024-12-06T14:44:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dd84621a57e54369849ddf5bff313774_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496261661/Put/seqid=0 2024-12-06T14:44:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496321762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733496261729, max timestamp=1733496261771Num uniques 859; 2024-12-06T14:44:21,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T14:44:21,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496261756, max timestamp=1733496261771Num uniques 859; 2024-12-06T14:44:21,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T14:44:21,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733496261756, max timestamp=1733496261771Num uniques 859; 2024-12-06T14:44:21,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bb7ea3e46d9c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T14:44:21,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742363_1539 (size=17034) 2024-12-06T14:44:21,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496321878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496321878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:21,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496321881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496321998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496321998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,065 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9828366e7dd24795a1e4942790f0edd8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9828366e7dd24795a1e4942790f0edd8 2024-12-06T14:44:22,070 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into 9828366e7dd24795a1e4942790f0edd8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:22,070 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:22,070 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496261552; duration=0sec 2024-12-06T14:44:22,070 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:22,070 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:22,071 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:22,074 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:22,074 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:22,074 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:22,074 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cb7181e6e821469b99396a3f15813ae9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5dab2d7e162441c5950f4821d5a0ee23, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5c13cf4c740d4ffd8b9e00b0abfdeb53] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=32.9 K 2024-12-06T14:44:22,075 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cb7181e6e821469b99396a3f15813ae9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733496258426 2024-12-06T14:44:22,075 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dab2d7e162441c5950f4821d5a0ee23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733496258448 2024-12-06T14:44:22,075 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c13cf4c740d4ffd8b9e00b0abfdeb53, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733496260616 2024-12-06T14:44:22,120 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/0f6e285f40d54b43b3e2835a95bde772 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/0f6e285f40d54b43b3e2835a95bde772 2024-12-06T14:44:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496322098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,126 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into 0f6e285f40d54b43b3e2835a95bde772(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:22,126 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:22,126 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496261551; duration=0sec 2024-12-06T14:44:22,126 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:22,126 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:22,134 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:22,135 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/644911a9e62e4f2089eea361975f350d is 50, key is test_row_0/C:col10/1733496260940/Put/seqid=0 2024-12-06T14:44:22,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742364_1540 (size=12104) 2024-12-06T14:44:22,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:22,182 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/644911a9e62e4f2089eea361975f350d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/644911a9e62e4f2089eea361975f350d 2024-12-06T14:44:22,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dd84621a57e54369849ddf5bff313774_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dd84621a57e54369849ddf5bff313774_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:22,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a0d144af582340018035e24c59965da8, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:22,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a0d144af582340018035e24c59965da8 is 175, key is test_row_0/A:col10/1733496261661/Put/seqid=0 2024-12-06T14:44:22,200 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into 644911a9e62e4f2089eea361975f350d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:22,200 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:22,200 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496261552; duration=0sec 2024-12-06T14:44:22,200 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:22,200 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:22,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496322209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496322209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742365_1541 (size=48139) 2024-12-06T14:44:22,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a0d144af582340018035e24c59965da8 2024-12-06T14:44:22,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/0b37c2aff8c944e29bf3ce7598b4ff00 is 50, key is test_row_0/B:col10/1733496261661/Put/seqid=0 2024-12-06T14:44:22,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742366_1542 (size=12001) 2024-12-06T14:44:22,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/0b37c2aff8c944e29bf3ce7598b4ff00 2024-12-06T14:44:22,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6fdfadfcc6d2437492bf446fb45fdba5 is 50, key is test_row_0/C:col10/1733496261661/Put/seqid=0 2024-12-06T14:44:22,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742367_1543 (size=12001) 2024-12-06T14:44:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496322429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6fdfadfcc6d2437492bf446fb45fdba5 2024-12-06T14:44:22,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a0d144af582340018035e24c59965da8 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8 2024-12-06T14:44:22,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8, entries=250, sequenceid=88, filesize=47.0 K 2024-12-06T14:44:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/0b37c2aff8c944e29bf3ce7598b4ff00 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/0b37c2aff8c944e29bf3ce7598b4ff00 2024-12-06T14:44:22,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/0b37c2aff8c944e29bf3ce7598b4ff00, entries=150, sequenceid=88, filesize=11.7 K 2024-12-06T14:44:22,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6fdfadfcc6d2437492bf446fb45fdba5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6fdfadfcc6d2437492bf446fb45fdba5 2024-12-06T14:44:22,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6fdfadfcc6d2437492bf446fb45fdba5, entries=150, sequenceid=88, filesize=11.7 K 2024-12-06T14:44:22,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=122.21 KB/125140 for 3623e1fd493fe297ada6ccd9ac6123ae in 835ms, sequenceid=88, compaction requested=false 2024-12-06T14:44:22,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:22,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=128.92 KB heapSize=368.31 KB 2024-12-06T14:44:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T14:44:22,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c4e8b8db203e4c62ad4ac96cbddec8b6_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T14:44:22,589 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:22,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-06T14:44:22,593 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:22,595 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:22,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496322648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496322648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,650 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:22,650 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496322649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496322649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742368_1544 (size=14594) 2024-12-06T14:44:22,653 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c4e8b8db203e4c62ad4ac96cbddec8b6_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c4e8b8db203e4c62ad4ac96cbddec8b6_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:22,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/456b45b293034171827256d2a5691835, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:22,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/456b45b293034171827256d2a5691835 is 175, key is test_row_0/A:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:22,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:22,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:22,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742369_1545 (size=39549) 2024-12-06T14:44:22,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496322773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496322774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:22,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:22,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:22,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496322959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496322998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496323008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/456b45b293034171827256d2a5691835 2024-12-06T14:44:23,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ed8a1996fb2145be94df5de88a96a0d0 is 50, key is test_row_0/B:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:23,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742370_1546 (size=12001) 2024-12-06T14:44:23,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ed8a1996fb2145be94df5de88a96a0d0 2024-12-06T14:44:23,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/e6195bb074cc4b80ab73c40dee96c018 is 50, key is test_row_0/C:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:23,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742371_1547 (size=12001) 2024-12-06T14:44:23,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/e6195bb074cc4b80ab73c40dee96c018 2024-12-06T14:44:23,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/456b45b293034171827256d2a5691835 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835 2024-12-06T14:44:23,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835, entries=200, sequenceid=127, filesize=38.6 K 2024-12-06T14:44:23,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ed8a1996fb2145be94df5de88a96a0d0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ed8a1996fb2145be94df5de88a96a0d0 2024-12-06T14:44:23,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ed8a1996fb2145be94df5de88a96a0d0, entries=150, sequenceid=127, filesize=11.7 K 2024-12-06T14:44:23,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/e6195bb074cc4b80ab73c40dee96c018 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/e6195bb074cc4b80ab73c40dee96c018 2024-12-06T14:44:23,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/e6195bb074cc4b80ab73c40dee96c018, entries=150, sequenceid=127, filesize=11.7 K 2024-12-06T14:44:23,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~128.92 KB/132010, heapSize ~368.27 KB/377104, currentSize=53.67 KB/54960 for 3623e1fd493fe297ada6ccd9ac6123ae in 704ms, sequenceid=127, compaction requested=true 2024-12-06T14:44:23,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:23,266 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:23,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T14:44:23,266 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118853 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:23,267 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,267 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,267 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9828366e7dd24795a1e4942790f0edd8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/0b37c2aff8c944e29bf3ce7598b4ff00, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ed8a1996fb2145be94df5de88a96a0d0] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=35.3 K 2024-12-06T14:44:23,267 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/0f6e285f40d54b43b3e2835a95bde772, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=116.1 K 2024-12-06T14:44:23,267 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/0f6e285f40d54b43b3e2835a95bde772, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835] 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9828366e7dd24795a1e4942790f0edd8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733496258457 2024-12-06T14:44:23,267 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6e285f40d54b43b3e2835a95bde772, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733496258457 2024-12-06T14:44:23,268 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d144af582340018035e24c59965da8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733496261650 2024-12-06T14:44:23,268 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b37c2aff8c944e29bf3ce7598b4ff00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733496261659 2024-12-06T14:44:23,268 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 456b45b293034171827256d2a5691835, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733496261829 2024-12-06T14:44:23,268 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8a1996fb2145be94df5de88a96a0d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733496261829 2024-12-06T14:44:23,273 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:23,275 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:23,275 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206ab7ff6d885f84ea398a091b5f81cc947_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:23,275 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/e76b2c96b8424a33a64d23d32ee80a53 is 50, key is test_row_0/B:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:23,277 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206ab7ff6d885f84ea398a091b5f81cc947_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:23,277 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ab7ff6d885f84ea398a091b5f81cc947_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:23,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742372_1548 (size=12207) 2024-12-06T14:44:23,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742373_1549 (size=4469) 2024-12-06T14:44:23,290 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#465 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:23,290 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/6e684d50b17a4071907e73f99be2842c is 175, key is test_row_0/A:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:23,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742374_1550 (size=31161) 2024-12-06T14:44:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:23,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T14:44:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:23,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120673ddc50911ad4be58afabac160e04353_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496262567/Put/seqid=0 2024-12-06T14:44:23,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742375_1551 (size=12304) 2024-12-06T14:44:23,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496323362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496323362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496323479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496323479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,689 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/e76b2c96b8424a33a64d23d32ee80a53 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e76b2c96b8424a33a64d23d32ee80a53 2024-12-06T14:44:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496323688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496323688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,693 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into e76b2c96b8424a33a64d23d32ee80a53(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:23,693 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:23,693 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496263266; duration=0sec 2024-12-06T14:44:23,693 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:23,693 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:23,693 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:23,694 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:23,694 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:23,694 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,694 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/644911a9e62e4f2089eea361975f350d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6fdfadfcc6d2437492bf446fb45fdba5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/e6195bb074cc4b80ab73c40dee96c018] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=35.3 K 2024-12-06T14:44:23,694 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 644911a9e62e4f2089eea361975f350d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733496258457 2024-12-06T14:44:23,694 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fdfadfcc6d2437492bf446fb45fdba5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733496261659 2024-12-06T14:44:23,694 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e6195bb074cc4b80ab73c40dee96c018, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733496261829 2024-12-06T14:44:23,699 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:23,699 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/ac32369681bf43e99d4968616fb17131 is 50, key is test_row_0/C:col10/1733496262560/Put/seqid=0 2024-12-06T14:44:23,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742376_1552 (size=12207) 2024-12-06T14:44:23,710 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/6e684d50b17a4071907e73f99be2842c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/6e684d50b17a4071907e73f99be2842c 2024-12-06T14:44:23,713 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into 6e684d50b17a4071907e73f99be2842c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:23,713 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:23,713 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496263265; duration=0sec 2024-12-06T14:44:23,713 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:23,713 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:23,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:23,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120673ddc50911ad4be58afabac160e04353_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120673ddc50911ad4be58afabac160e04353_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:23,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/8c5afa761e344d899d9935902f0efa08, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:23,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/8c5afa761e344d899d9935902f0efa08 is 175, key is test_row_0/A:col10/1733496262567/Put/seqid=0 2024-12-06T14:44:23,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742377_1553 (size=31105) 2024-12-06T14:44:23,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/8c5afa761e344d899d9935902f0efa08 2024-12-06T14:44:23,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/be3925b830a94f1f893d0630d1d49373 is 50, key is test_row_0/B:col10/1733496262567/Put/seqid=0 2024-12-06T14:44:23,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742378_1554 (size=12151) 2024-12-06T14:44:23,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:23,981 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:23,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:23,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:24,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496323998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496323998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,105 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/ac32369681bf43e99d4968616fb17131 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac32369681bf43e99d4968616fb17131 2024-12-06T14:44:24,108 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into ac32369681bf43e99d4968616fb17131(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:24,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:24,108 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496263266; duration=0sec 2024-12-06T14:44:24,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:24,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:24,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:24,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:24,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/be3925b830a94f1f893d0630d1d49373 2024-12-06T14:44:24,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/24de46c7fac94cb6a83ad6706e43c40a is 50, key is test_row_0/C:col10/1733496262567/Put/seqid=0 2024-12-06T14:44:24,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742379_1555 (size=12151) 2024-12-06T14:44:24,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/24de46c7fac94cb6a83ad6706e43c40a 2024-12-06T14:44:24,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/8c5afa761e344d899d9935902f0efa08 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08 2024-12-06T14:44:24,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08, entries=150, sequenceid=139, filesize=30.4 K 2024-12-06T14:44:24,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/be3925b830a94f1f893d0630d1d49373 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/be3925b830a94f1f893d0630d1d49373 2024-12-06T14:44:24,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/be3925b830a94f1f893d0630d1d49373, entries=150, sequenceid=139, filesize=11.9 K 2024-12-06T14:44:24,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/24de46c7fac94cb6a83ad6706e43c40a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/24de46c7fac94cb6a83ad6706e43c40a 2024-12-06T14:44:24,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/24de46c7fac94cb6a83ad6706e43c40a, entries=150, sequenceid=139, filesize=11.9 K 2024-12-06T14:44:24,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3623e1fd493fe297ada6ccd9ac6123ae in 884ms, sequenceid=139, compaction requested=false 2024-12-06T14:44:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:24,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T14:44:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:24,294 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T14:44:24,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:24,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:24,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:24,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:24,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:24,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206998ae1c549874fe1850588332dd2373c_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496263354/Put/seqid=0 2024-12-06T14:44:24,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742380_1556 (size=12304) 2024-12-06T14:44:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:24,317 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206998ae1c549874fe1850588332dd2373c_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206998ae1c549874fe1850588332dd2373c_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:24,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/02c842e3ba7241d58bf53324e1a03e0d, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/02c842e3ba7241d58bf53324e1a03e0d is 175, key is test_row_0/A:col10/1733496263354/Put/seqid=0 2024-12-06T14:44:24,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742381_1557 (size=31105) 2024-12-06T14:44:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:24,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496324522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496324524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496324628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496324638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,727 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/02c842e3ba7241d58bf53324e1a03e0d 2024-12-06T14:44:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ddcb0d5279e444ad9766ecf69a69a988 is 50, key is test_row_0/B:col10/1733496263354/Put/seqid=0 2024-12-06T14:44:24,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742382_1558 (size=12151) 2024-12-06T14:44:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496324838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496324848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:25,135 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ddcb0d5279e444ad9766ecf69a69a988 2024-12-06T14:44:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf88c86deca640b1a10a2b04e666d2c0 is 50, key is test_row_0/C:col10/1733496263354/Put/seqid=0 2024-12-06T14:44:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742383_1559 (size=12151) 2024-12-06T14:44:25,144 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf88c86deca640b1a10a2b04e666d2c0 2024-12-06T14:44:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/02c842e3ba7241d58bf53324e1a03e0d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d 2024-12-06T14:44:25,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496325148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:25,151 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d, entries=150, sequenceid=162, filesize=30.4 K 2024-12-06T14:44:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ddcb0d5279e444ad9766ecf69a69a988 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ddcb0d5279e444ad9766ecf69a69a988 2024-12-06T14:44:25,155 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ddcb0d5279e444ad9766ecf69a69a988, entries=150, sequenceid=162, filesize=11.9 K 2024-12-06T14:44:25,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf88c86deca640b1a10a2b04e666d2c0 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf88c86deca640b1a10a2b04e666d2c0 2024-12-06T14:44:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:25,170 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf88c86deca640b1a10a2b04e666d2c0, entries=150, sequenceid=162, filesize=11.9 K 2024-12-06T14:44:25,171 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3623e1fd493fe297ada6ccd9ac6123ae in 877ms, sequenceid=162, compaction requested=true 2024-12-06T14:44:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-06T14:44:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-06T14:44:25,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-06T14:44:25,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5770 sec 2024-12-06T14:44:25,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.5860 sec 2024-12-06T14:44:25,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T14:44:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:25,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a210629e265a4968b394406d8f47b5ec_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:25,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742384_1560 (size=14794) 2024-12-06T14:44:25,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:25,687 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a210629e265a4968b394406d8f47b5ec_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a210629e265a4968b394406d8f47b5ec_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:25,688 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de9919b4388f48c1a8cc1d0926515372, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:25,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de9919b4388f48c1a8cc1d0926515372 is 175, key is test_row_0/A:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:25,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742385_1561 (size=39749) 2024-12-06T14:44:25,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496325702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:25,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496325706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496325819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496325819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496325998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,000 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:26,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496326028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496326028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,095 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de9919b4388f48c1a8cc1d0926515372 2024-12-06T14:44:26,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/f72eae70d41249f994d823e656a7f199 is 50, key is test_row_0/B:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:26,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742386_1562 (size=12151) 2024-12-06T14:44:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496326338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496326338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/f72eae70d41249f994d823e656a7f199 2024-12-06T14:44:26,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/aad09bdce7f34a66956fb0a92e302272 is 50, key is test_row_0/C:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:26,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742387_1563 (size=12151) 2024-12-06T14:44:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496326678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496326678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,681 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,682 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_2' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T14:44:26,768 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:26,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-06T14:44:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T14:44:26,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:26,771 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:26,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:26,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496326848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496326848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T14:44:26,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T14:44:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:26,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:26,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/aad09bdce7f34a66956fb0a92e302272 2024-12-06T14:44:26,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de9919b4388f48c1a8cc1d0926515372 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372 2024-12-06T14:44:26,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372, entries=200, sequenceid=175, filesize=38.8 K 2024-12-06T14:44:26,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/f72eae70d41249f994d823e656a7f199 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/f72eae70d41249f994d823e656a7f199 2024-12-06T14:44:26,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/f72eae70d41249f994d823e656a7f199, entries=150, sequenceid=175, filesize=11.9 K 2024-12-06T14:44:26,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/aad09bdce7f34a66956fb0a92e302272 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/aad09bdce7f34a66956fb0a92e302272 2024-12-06T14:44:26,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/aad09bdce7f34a66956fb0a92e302272, entries=150, sequenceid=175, filesize=11.9 K 2024-12-06T14:44:26,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3623e1fd493fe297ada6ccd9ac6123ae in 1287ms, sequenceid=175, compaction requested=true 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:26,949 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:26,949 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:26,950 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:26,950 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:26,950 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:26,950 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/6e684d50b17a4071907e73f99be2842c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=130 K 2024-12-06T14:44:26,950 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:26,950 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/6e684d50b17a4071907e73f99be2842c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372] 2024-12-06T14:44:26,951 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:26,951 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e684d50b17a4071907e73f99be2842c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733496261829 2024-12-06T14:44:26,951 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:26,951 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:26,951 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e76b2c96b8424a33a64d23d32ee80a53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/be3925b830a94f1f893d0630d1d49373, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ddcb0d5279e444ad9766ecf69a69a988, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/f72eae70d41249f994d823e656a7f199] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=47.5 K 2024-12-06T14:44:26,951 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e76b2c96b8424a33a64d23d32ee80a53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733496261829 2024-12-06T14:44:26,951 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5afa761e344d899d9935902f0efa08, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733496262567 2024-12-06T14:44:26,952 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting be3925b830a94f1f893d0630d1d49373, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733496262567 2024-12-06T14:44:26,952 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02c842e3ba7241d58bf53324e1a03e0d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733496263352 2024-12-06T14:44:26,952 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9919b4388f48c1a8cc1d0926515372, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733496264519 2024-12-06T14:44:26,952 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ddcb0d5279e444ad9766ecf69a69a988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733496263352 2024-12-06T14:44:26,952 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting f72eae70d41249f994d823e656a7f199, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733496264519 2024-12-06T14:44:26,957 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:26,959 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:26,959 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/fbbdf528df924391a6a91b41c3dc2070 is 50, key is test_row_0/B:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:26,959 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120639c6f257f2db4d6ca5ff6cb089c3189b_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:26,961 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120639c6f257f2db4d6ca5ff6cb089c3189b_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:26,961 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120639c6f257f2db4d6ca5ff6cb089c3189b_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:26,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742388_1564 (size=12493) 2024-12-06T14:44:26,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742389_1565 (size=4469) 2024-12-06T14:44:26,976 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#478 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:26,977 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b48567c80ac4416096155293c4ef81c7 is 175, key is test_row_0/A:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:26,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742390_1566 (size=31447) 2024-12-06T14:44:27,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T14:44:27,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:27,076 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:44:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061083e52c0b0e43c2b53c05a9a01ccef6_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496265704/Put/seqid=0 2024-12-06T14:44:27,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742391_1567 (size=12304) 2024-12-06T14:44:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T14:44:27,377 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/fbbdf528df924391a6a91b41c3dc2070 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/fbbdf528df924391a6a91b41c3dc2070 2024-12-06T14:44:27,380 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into fbbdf528df924391a6a91b41c3dc2070(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:27,380 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:27,380 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=12, startTime=1733496266949; duration=0sec 2024-12-06T14:44:27,380 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:27,380 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:27,380 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T14:44:27,381 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T14:44:27,382 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:27,382 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:27,382 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac32369681bf43e99d4968616fb17131, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/24de46c7fac94cb6a83ad6706e43c40a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf88c86deca640b1a10a2b04e666d2c0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/aad09bdce7f34a66956fb0a92e302272] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=47.5 K 2024-12-06T14:44:27,382 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ac32369681bf43e99d4968616fb17131, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733496261829 2024-12-06T14:44:27,383 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 24de46c7fac94cb6a83ad6706e43c40a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733496262567 2024-12-06T14:44:27,383 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cf88c86deca640b1a10a2b04e666d2c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733496263352 2024-12-06T14:44:27,383 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting aad09bdce7f34a66956fb0a92e302272, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733496264519 2024-12-06T14:44:27,385 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b48567c80ac4416096155293c4ef81c7 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b48567c80ac4416096155293c4ef81c7 2024-12-06T14:44:27,389 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into b48567c80ac4416096155293c4ef81c7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:27,389 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:27,389 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=12, startTime=1733496266949; duration=0sec 2024-12-06T14:44:27,389 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:27,389 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:27,390 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:27,390 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/2168fb45ff4048278030d75c5abcc699 is 50, key is test_row_0/C:col10/1733496264521/Put/seqid=0 2024-12-06T14:44:27,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742392_1568 (size=12493) 2024-12-06T14:44:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T14:44:27,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:27,487 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061083e52c0b0e43c2b53c05a9a01ccef6_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061083e52c0b0e43c2b53c05a9a01ccef6_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:27,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/41dd0044f3374abfb76ae75d52fa47c6, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:27,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/41dd0044f3374abfb76ae75d52fa47c6 is 175, key is test_row_0/A:col10/1733496265704/Put/seqid=0 2024-12-06T14:44:27,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742393_1569 (size=31105) 2024-12-06T14:44:27,492 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/41dd0044f3374abfb76ae75d52fa47c6 2024-12-06T14:44:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/d230783ee625471a943817e863b36ab3 is 50, key is test_row_0/B:col10/1733496265704/Put/seqid=0 2024-12-06T14:44:27,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742394_1570 (size=12151) 2024-12-06T14:44:27,798 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/2168fb45ff4048278030d75c5abcc699 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/2168fb45ff4048278030d75c5abcc699 2024-12-06T14:44:27,804 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into 2168fb45ff4048278030d75c5abcc699(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:27,804 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:27,804 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=12, startTime=1733496266949; duration=0sec 2024-12-06T14:44:27,804 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:27,804 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496327890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496327890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:27,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:27,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:27,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:27,904 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/d230783ee625471a943817e863b36ab3 2024-12-06T14:44:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T14:44:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/93a084cb40564ba2939c2e20eebd8535 is 50, key is test_row_0/C:col10/1733496265704/Put/seqid=0 2024-12-06T14:44:27,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742395_1571 (size=12151) 2024-12-06T14:44:27,928 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/93a084cb40564ba2939c2e20eebd8535 2024-12-06T14:44:27,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/41dd0044f3374abfb76ae75d52fa47c6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6 2024-12-06T14:44:27,938 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6, entries=150, sequenceid=194, filesize=30.4 K 2024-12-06T14:44:27,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/d230783ee625471a943817e863b36ab3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/d230783ee625471a943817e863b36ab3 2024-12-06T14:44:27,944 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/d230783ee625471a943817e863b36ab3, entries=150, sequenceid=194, filesize=11.9 K 2024-12-06T14:44:27,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/93a084cb40564ba2939c2e20eebd8535 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/93a084cb40564ba2939c2e20eebd8535 2024-12-06T14:44:27,948 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/93a084cb40564ba2939c2e20eebd8535, entries=150, sequenceid=194, filesize=11.9 K 2024-12-06T14:44:27,949 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 3623e1fd493fe297ada6ccd9ac6123ae in 873ms, sequenceid=194, compaction requested=false 2024-12-06T14:44:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-06T14:44:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-06T14:44:27,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-06T14:44:27,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1790 sec 2024-12-06T14:44:27,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.1850 sec 2024-12-06T14:44:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:28,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:44:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:28,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206049e148da147456490b04070b53f6454_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:28,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742396_1572 (size=17284) 2024-12-06T14:44:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496328038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496328046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496328148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496328158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496328359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496328368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:28,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206049e148da147456490b04070b53f6454_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206049e148da147456490b04070b53f6454_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:28,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de3b784beae840fab46b7c54c6ee315c, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:28,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de3b784beae840fab46b7c54c6ee315c is 175, key is test_row_0/A:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:28,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742397_1573 (size=48389) 2024-12-06T14:44:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496328668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496328678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:28,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de3b784beae840fab46b7c54c6ee315c 2024-12-06T14:44:28,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/da4bd412358c48429e3d0c0a41dd8cd3 is 50, key is test_row_0/B:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:28,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742398_1574 (size=12151) 2024-12-06T14:44:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T14:44:28,918 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:28,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-06T14:44:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T14:44:28,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:28,921 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:28,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T14:44:29,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T14:44:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496329178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:29,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496329188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:29,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T14:44:29,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:29,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T14:44:29,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/da4bd412358c48429e3d0c0a41dd8cd3 2024-12-06T14:44:29,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6249e492e1684fb884fb779b8844227d is 50, key is test_row_0/C:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742399_1575 (size=12151) 2024-12-06T14:44:29,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T14:44:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T14:44:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T14:44:29,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6249e492e1684fb884fb779b8844227d 2024-12-06T14:44:29,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de3b784beae840fab46b7c54c6ee315c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c 2024-12-06T14:44:29,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c, entries=250, sequenceid=213, filesize=47.3 K 2024-12-06T14:44:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/da4bd412358c48429e3d0c0a41dd8cd3 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/da4bd412358c48429e3d0c0a41dd8cd3 2024-12-06T14:44:29,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/da4bd412358c48429e3d0c0a41dd8cd3, entries=150, sequenceid=213, filesize=11.9 K 2024-12-06T14:44:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6249e492e1684fb884fb779b8844227d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6249e492e1684fb884fb779b8844227d 2024-12-06T14:44:29,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6249e492e1684fb884fb779b8844227d, entries=150, sequenceid=213, filesize=11.9 K 2024-12-06T14:44:29,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 3623e1fd493fe297ada6ccd9ac6123ae in 1680ms, sequenceid=213, compaction requested=true 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:29,681 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:29,681 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:29,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:29,682 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:29,682 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:29,682 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:29,682 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:29,682 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,682 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,682 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/fbbdf528df924391a6a91b41c3dc2070, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/d230783ee625471a943817e863b36ab3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/da4bd412358c48429e3d0c0a41dd8cd3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=35.9 K 2024-12-06T14:44:29,682 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b48567c80ac4416096155293c4ef81c7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=108.3 K 2024-12-06T14:44:29,682 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,682 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b48567c80ac4416096155293c4ef81c7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c] 2024-12-06T14:44:29,682 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbdf528df924391a6a91b41c3dc2070, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733496264519 2024-12-06T14:44:29,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T14:44:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,683 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:44:29,683 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48567c80ac4416096155293c4ef81c7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733496264519 2024-12-06T14:44:29,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d230783ee625471a943817e863b36ab3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496265695 2024-12-06T14:44:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:29,683 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41dd0044f3374abfb76ae75d52fa47c6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496265695 2024-12-06T14:44:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:29,683 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting da4bd412358c48429e3d0c0a41dd8cd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733496267876 2024-12-06T14:44:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:29,683 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting de3b784beae840fab46b7c54c6ee315c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733496267876 2024-12-06T14:44:29,687 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:29,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206949a956ded9c4bc0984eaa203d7614be_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496268032/Put/seqid=0 2024-12-06T14:44:29,688 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:29,688 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ce6bb0619bd04d9fa9ac53da190c6c06 is 50, key is test_row_0/B:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:29,688 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412062bf2487cff064ff5b50f7863d3659c04_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:29,690 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412062bf2487cff064ff5b50f7863d3659c04_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:29,690 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062bf2487cff064ff5b50f7863d3659c04_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:29,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742400_1576 (size=12304) 2024-12-06T14:44:29,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742401_1577 (size=12595) 2024-12-06T14:44:29,695 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/ce6bb0619bd04d9fa9ac53da190c6c06 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ce6bb0619bd04d9fa9ac53da190c6c06 2024-12-06T14:44:29,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742402_1578 (size=4469) 2024-12-06T14:44:29,699 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into ce6bb0619bd04d9fa9ac53da190c6c06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:29,699 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:29,699 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496269681; duration=0sec 2024-12-06T14:44:29,699 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:29,699 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:29,699 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:29,700 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:29,700 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:29,700 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:29,700 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/2168fb45ff4048278030d75c5abcc699, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/93a084cb40564ba2939c2e20eebd8535, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6249e492e1684fb884fb779b8844227d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=35.9 K 2024-12-06T14:44:29,700 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 2168fb45ff4048278030d75c5abcc699, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733496264519 2024-12-06T14:44:29,700 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a084cb40564ba2939c2e20eebd8535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733496265695 2024-12-06T14:44:29,701 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6249e492e1684fb884fb779b8844227d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733496267876 2024-12-06T14:44:29,705 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:29,705 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/d4db40032f5c40d7b9fa86659d4dcf7c is 50, key is test_row_0/C:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742403_1579 (size=12595) 2024-12-06T14:44:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T14:44:30,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496330071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:30,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:30,094 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206949a956ded9c4bc0984eaa203d7614be_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206949a956ded9c4bc0984eaa203d7614be_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a95c06714c634ba7b36ada45a2c2eea1, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a95c06714c634ba7b36ada45a2c2eea1 is 175, key is test_row_0/A:col10/1733496268032/Put/seqid=0 2024-12-06T14:44:30,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742404_1580 (size=31105) 2024-12-06T14:44:30,099 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#488 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:30,099 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/1ed8e281f7d74eaa9cb24d9c93c54011 is 175, key is test_row_0/A:col10/1733496268000/Put/seqid=0 2024-12-06T14:44:30,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742405_1581 (size=31549) 2024-12-06T14:44:30,111 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/d4db40032f5c40d7b9fa86659d4dcf7c as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d4db40032f5c40d7b9fa86659d4dcf7c 2024-12-06T14:44:30,115 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into d4db40032f5c40d7b9fa86659d4dcf7c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:30,115 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:30,115 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496269681; duration=0sec 2024-12-06T14:44:30,115 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:30,115 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:30,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496330178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:30,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:30,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496330188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:30,190 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:30,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:30,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496330198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:30,199 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:30,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496330388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:30,498 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a95c06714c634ba7b36ada45a2c2eea1 2024-12-06T14:44:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/46fe172181cc4b4a9d22eaf5cdab4408 is 50, key is test_row_0/B:col10/1733496268032/Put/seqid=0 2024-12-06T14:44:30,507 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/1ed8e281f7d74eaa9cb24d9c93c54011 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1ed8e281f7d74eaa9cb24d9c93c54011 2024-12-06T14:44:30,515 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into 1ed8e281f7d74eaa9cb24d9c93c54011(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:30,515 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:30,515 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496269681; duration=0sec 2024-12-06T14:44:30,515 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:30,515 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:30,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742406_1582 (size=12151) 2024-12-06T14:44:30,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496330698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:30,926 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/46fe172181cc4b4a9d22eaf5cdab4408 2024-12-06T14:44:30,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/3425569c8db749a098ae71768a6b2e84 is 50, key is test_row_0/C:col10/1733496268032/Put/seqid=0 2024-12-06T14:44:30,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742407_1583 (size=12151) 2024-12-06T14:44:30,990 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/3425569c8db749a098ae71768a6b2e84 2024-12-06T14:44:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a95c06714c634ba7b36ada45a2c2eea1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1 2024-12-06T14:44:30,997 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1, entries=150, sequenceid=230, filesize=30.4 K 2024-12-06T14:44:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/46fe172181cc4b4a9d22eaf5cdab4408 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/46fe172181cc4b4a9d22eaf5cdab4408 2024-12-06T14:44:31,002 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/46fe172181cc4b4a9d22eaf5cdab4408, entries=150, sequenceid=230, filesize=11.9 K 2024-12-06T14:44:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/3425569c8db749a098ae71768a6b2e84 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3425569c8db749a098ae71768a6b2e84 2024-12-06T14:44:31,007 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3425569c8db749a098ae71768a6b2e84, entries=150, sequenceid=230, filesize=11.9 K 2024-12-06T14:44:31,008 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 3623e1fd493fe297ada6ccd9ac6123ae in 1325ms, sequenceid=230, compaction requested=false 2024-12-06T14:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-06T14:44:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-06T14:44:31,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-06T14:44:31,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0890 sec 2024-12-06T14:44:31,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0930 sec 2024-12-06T14:44:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T14:44:31,069 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:31,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-06T14:44:31,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T14:44:31,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:31,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T14:44:31,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T14:44:31,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:31,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:31,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:31,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T14:44:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:31,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065b77cfa100dd4e9a83dd5b59c5482e1e_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742408_1584 (size=14794) 2024-12-06T14:44:31,280 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:31,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065b77cfa100dd4e9a83dd5b59c5482e1e_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065b77cfa100dd4e9a83dd5b59c5482e1e_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:31,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/65efdb56801543fe9ec89d6f3aca4ff9, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:31,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/65efdb56801543fe9ec89d6f3aca4ff9 is 175, key is test_row_0/A:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496331291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742409_1585 (size=39749) 2024-12-06T14:44:31,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/65efdb56801543fe9ec89d6f3aca4ff9 2024-12-06T14:44:31,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/4e4a1ec5b5e94b84992a9eb54ddec5dc is 50, key is test_row_0/B:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742410_1586 (size=12151) 2024-12-06T14:44:31,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T14:44:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:31,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/4e4a1ec5b5e94b84992a9eb54ddec5dc 2024-12-06T14:44:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T14:44:31,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/b44bf668837146b5975281e805a32ffb is 50, key is test_row_0/C:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496331398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:31,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742411_1587 (size=12151) 2024-12-06T14:44:31,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/b44bf668837146b5975281e805a32ffb 2024-12-06T14:44:31,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/65efdb56801543fe9ec89d6f3aca4ff9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9 2024-12-06T14:44:31,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9, entries=200, sequenceid=249, filesize=38.8 K 2024-12-06T14:44:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/4e4a1ec5b5e94b84992a9eb54ddec5dc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4e4a1ec5b5e94b84992a9eb54ddec5dc 2024-12-06T14:44:31,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4e4a1ec5b5e94b84992a9eb54ddec5dc, entries=150, sequenceid=249, filesize=11.9 K 2024-12-06T14:44:31,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/b44bf668837146b5975281e805a32ffb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b44bf668837146b5975281e805a32ffb 2024-12-06T14:44:31,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b44bf668837146b5975281e805a32ffb, entries=150, sequenceid=249, filesize=11.9 K 2024-12-06T14:44:31,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3623e1fd493fe297ada6ccd9ac6123ae in 216ms, sequenceid=249, compaction requested=true 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:31,426 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T14:44:31,426 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:31,427 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:31,427 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:31,427 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:31,427 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:31,427 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,428 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1ed8e281f7d74eaa9cb24d9c93c54011, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=100.0 K 2024-12-06T14:44:31,428 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,428 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1ed8e281f7d74eaa9cb24d9c93c54011, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9] 2024-12-06T14:44:31,428 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,428 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ce6bb0619bd04d9fa9ac53da190c6c06, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/46fe172181cc4b4a9d22eaf5cdab4408, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4e4a1ec5b5e94b84992a9eb54ddec5dc] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.0 K 2024-12-06T14:44:31,429 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed8e281f7d74eaa9cb24d9c93c54011, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733496267876 2024-12-06T14:44:31,429 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ce6bb0619bd04d9fa9ac53da190c6c06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733496267876 2024-12-06T14:44:31,430 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95c06714c634ba7b36ada45a2c2eea1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733496268032 2024-12-06T14:44:31,430 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 46fe172181cc4b4a9d22eaf5cdab4408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733496268032 2024-12-06T14:44:31,430 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65efdb56801543fe9ec89d6f3aca4ff9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733496270050 2024-12-06T14:44:31,430 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e4a1ec5b5e94b84992a9eb54ddec5dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733496270061 2024-12-06T14:44:31,442 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:31,443 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/cd4ff6dde8b2450b86414cee2e09d36e is 50, key is test_row_0/B:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,450 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:31,468 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206e5359bcfef0545d0bf0f839e9bd3532b_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:31,470 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206e5359bcfef0545d0bf0f839e9bd3532b_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:31,470 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e5359bcfef0545d0bf0f839e9bd3532b_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:31,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742412_1588 (size=12697) 2024-12-06T14:44:31,496 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/cd4ff6dde8b2450b86414cee2e09d36e as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/cd4ff6dde8b2450b86414cee2e09d36e 2024-12-06T14:44:31,504 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into cd4ff6dde8b2450b86414cee2e09d36e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:31,504 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:31,504 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496271426; duration=0sec 2024-12-06T14:44:31,504 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:31,504 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:31,504 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:31,506 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:31,507 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:31,507 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,507 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d4db40032f5c40d7b9fa86659d4dcf7c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3425569c8db749a098ae71768a6b2e84, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b44bf668837146b5975281e805a32ffb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.0 K 2024-12-06T14:44:31,508 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting d4db40032f5c40d7b9fa86659d4dcf7c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733496267876 2024-12-06T14:44:31,508 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 3425569c8db749a098ae71768a6b2e84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733496268032 2024-12-06T14:44:31,508 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b44bf668837146b5975281e805a32ffb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733496270061 2024-12-06T14:44:31,517 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:31,518 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf397c4e92044ca8a353fa38702c8497 is 50, key is test_row_0/C:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742413_1589 (size=4469) 2024-12-06T14:44:31,526 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#496 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:31,526 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/8a3261ffbc6141c689e15db0448e16e5 is 175, key is test_row_0/A:col10/1733496270061/Put/seqid=0 2024-12-06T14:44:31,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T14:44:31,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:31,532 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T14:44:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:31,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742415_1591 (size=12697) 2024-12-06T14:44:31,583 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf397c4e92044ca8a353fa38702c8497 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf397c4e92044ca8a353fa38702c8497 2024-12-06T14:44:31,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e1255627cba3477e85f2142405585452_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496271253/Put/seqid=0 2024-12-06T14:44:31,590 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into cf397c4e92044ca8a353fa38702c8497(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:31,590 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:31,590 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496271426; duration=0sec 2024-12-06T14:44:31,590 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:31,590 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:31,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742414_1590 (size=31651) 2024-12-06T14:44:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742416_1592 (size=12454) 2024-12-06T14:44:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:31,621 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e1255627cba3477e85f2142405585452_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e1255627cba3477e85f2142405585452_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:31,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/51a6241fbc9f45c08f2d970ca6c1cf54, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:31,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/51a6241fbc9f45c08f2d970ca6c1cf54 is 175, key is test_row_0/A:col10/1733496271253/Put/seqid=0 2024-12-06T14:44:31,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742417_1593 (size=31255) 2024-12-06T14:44:31,656 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/51a6241fbc9f45c08f2d970ca6c1cf54 2024-12-06T14:44:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/a02b2cccb79846fd90136cc15f8df561 is 50, key is test_row_0/B:col10/1733496271253/Put/seqid=0 2024-12-06T14:44:31,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496331668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:31,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742418_1594 (size=12301) 2024-12-06T14:44:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T14:44:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496331778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:31,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496331988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:32,001 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/8a3261ffbc6141c689e15db0448e16e5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8a3261ffbc6141c689e15db0448e16e5 2024-12-06T14:44:32,004 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into 8a3261ffbc6141c689e15db0448e16e5(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:32,004 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:32,004 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496271426; duration=0sec 2024-12-06T14:44:32,004 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:32,004 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:32,073 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/a02b2cccb79846fd90136cc15f8df561 2024-12-06T14:44:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/fc4064859eb04151a409c4371ddbecc1 is 50, key is test_row_0/C:col10/1733496271253/Put/seqid=0 2024-12-06T14:44:32,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742419_1595 (size=12301) 2024-12-06T14:44:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T14:44:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496332209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:32,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496332209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:32,210 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:32,210 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:32,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496332299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:32,483 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/fc4064859eb04151a409c4371ddbecc1 2024-12-06T14:44:32,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/51a6241fbc9f45c08f2d970ca6c1cf54 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54 2024-12-06T14:44:32,490 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54, entries=150, sequenceid=266, filesize=30.5 K 2024-12-06T14:44:32,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/a02b2cccb79846fd90136cc15f8df561 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a02b2cccb79846fd90136cc15f8df561 2024-12-06T14:44:32,495 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a02b2cccb79846fd90136cc15f8df561, entries=150, sequenceid=266, filesize=12.0 K 2024-12-06T14:44:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/fc4064859eb04151a409c4371ddbecc1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/fc4064859eb04151a409c4371ddbecc1 2024-12-06T14:44:32,499 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/fc4064859eb04151a409c4371ddbecc1, entries=150, sequenceid=266, filesize=12.0 K 2024-12-06T14:44:32,499 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3623e1fd493fe297ada6ccd9ac6123ae in 967ms, sequenceid=266, compaction requested=false 2024-12-06T14:44:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-06T14:44:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-06T14:44:32,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-06T14:44:32,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-12-06T14:44:32,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4330 sec 2024-12-06T14:44:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:32,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T14:44:32,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:32,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:32,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:32,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:32,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:32,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:32,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dca718031929434bbffe72a0e2a3d05f_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:32,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742420_1596 (size=14994) 2024-12-06T14:44:32,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:32,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dca718031929434bbffe72a0e2a3d05f_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dca718031929434bbffe72a0e2a3d05f_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:32,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a7afe75aa7474219b97dafe04b98cae5, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:32,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a7afe75aa7474219b97dafe04b98cae5 is 175, key is test_row_0/A:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:32,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742421_1597 (size=39949) 2024-12-06T14:44:32,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496332844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:32,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496332958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:33,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:33,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496333168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T14:44:33,219 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:33,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-06T14:44:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T14:44:33,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:33,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:33,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:33,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a7afe75aa7474219b97dafe04b98cae5 2024-12-06T14:44:33,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/bb1988ef526447139d2c54564030b6d9 is 50, key is test_row_0/B:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:33,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742422_1598 (size=12301) 2024-12-06T14:44:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T14:44:33,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T14:44:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:33,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496333478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:33,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T14:44:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T14:44:33,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/bb1988ef526447139d2c54564030b6d9 2024-12-06T14:44:33,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/b9d910e71ec24989b79480a412e066fc is 50, key is test_row_0/C:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:33,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742423_1599 (size=12301) 2024-12-06T14:44:33,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/b9d910e71ec24989b79480a412e066fc 2024-12-06T14:44:33,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a7afe75aa7474219b97dafe04b98cae5 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5 2024-12-06T14:44:33,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5, entries=200, sequenceid=285, filesize=39.0 K 2024-12-06T14:44:33,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/bb1988ef526447139d2c54564030b6d9 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/bb1988ef526447139d2c54564030b6d9 2024-12-06T14:44:33,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/bb1988ef526447139d2c54564030b6d9, entries=150, sequenceid=285, filesize=12.0 K 2024-12-06T14:44:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/b9d910e71ec24989b79480a412e066fc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b9d910e71ec24989b79480a412e066fc 2024-12-06T14:44:33,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b9d910e71ec24989b79480a412e066fc, entries=150, sequenceid=285, filesize=12.0 K 2024-12-06T14:44:33,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 3623e1fd493fe297ada6ccd9ac6123ae in 857ms, sequenceid=285, compaction requested=true 2024-12-06T14:44:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:33,667 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:33,667 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:33,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:33,668 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:33,668 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:33,668 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:33,668 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:33,668 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,668 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,668 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/cd4ff6dde8b2450b86414cee2e09d36e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a02b2cccb79846fd90136cc15f8df561, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/bb1988ef526447139d2c54564030b6d9] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.4 K 2024-12-06T14:44:33,668 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8a3261ffbc6141c689e15db0448e16e5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=100.4 K 2024-12-06T14:44:33,668 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,668 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8a3261ffbc6141c689e15db0448e16e5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5] 2024-12-06T14:44:33,669 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4ff6dde8b2450b86414cee2e09d36e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733496270061 2024-12-06T14:44:33,670 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3261ffbc6141c689e15db0448e16e5, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733496270061 2024-12-06T14:44:33,672 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting a02b2cccb79846fd90136cc15f8df561, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733496271253 2024-12-06T14:44:33,672 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a6241fbc9f45c08f2d970ca6c1cf54, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733496271253 2024-12-06T14:44:33,672 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7afe75aa7474219b97dafe04b98cae5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733496271644 2024-12-06T14:44:33,672 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting bb1988ef526447139d2c54564030b6d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733496271644 2024-12-06T14:44:33,676 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:33,677 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:33,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T14:44:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:33,678 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T14:44:33,678 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/b160a295132546e3a772507030f27dff is 50, key is test_row_0/B:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:33,686 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206a529a503180349cf99682a9e906c269b_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:33,688 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206a529a503180349cf99682a9e906c269b_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:33,688 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a529a503180349cf99682a9e906c269b_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:33,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742424_1600 (size=12949) 2024-12-06T14:44:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064dece1a8f9cf439bb28ad709b2c7aed2_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496272829/Put/seqid=0 2024-12-06T14:44:33,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742425_1601 (size=4469) 2024-12-06T14:44:33,711 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#505 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:33,712 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a85440c018bc4066802523e576b01177 is 175, key is test_row_0/A:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:33,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742426_1602 (size=12454) 2024-12-06T14:44:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:33,723 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064dece1a8f9cf439bb28ad709b2c7aed2_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064dece1a8f9cf439bb28ad709b2c7aed2_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:33,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/177f635aa1da4f6b9ebae6c50583e0eb, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:33,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/177f635aa1da4f6b9ebae6c50583e0eb is 175, key is test_row_0/A:col10/1733496272829/Put/seqid=0 2024-12-06T14:44:33,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742427_1603 (size=31903) 2024-12-06T14:44:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742428_1604 (size=31255) 2024-12-06T14:44:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T14:44:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:34,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496334052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:34,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:34,105 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/b160a295132546e3a772507030f27dff as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/b160a295132546e3a772507030f27dff 2024-12-06T14:44:34,108 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into b160a295132546e3a772507030f27dff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:34,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:34,108 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496273667; duration=0sec 2024-12-06T14:44:34,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:34,108 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:34,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:34,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:34,109 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:34,109 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:34,109 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf397c4e92044ca8a353fa38702c8497, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/fc4064859eb04151a409c4371ddbecc1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b9d910e71ec24989b79480a412e066fc] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.4 K 2024-12-06T14:44:34,110 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cf397c4e92044ca8a353fa38702c8497, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733496270061 2024-12-06T14:44:34,110 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4064859eb04151a409c4371ddbecc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733496271253 2024-12-06T14:44:34,110 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d910e71ec24989b79480a412e066fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733496271644 2024-12-06T14:44:34,116 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:34,116 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/9110842258964f53a0d95da554e13fc6 is 50, key is test_row_0/C:col10/1733496272809/Put/seqid=0 2024-12-06T14:44:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742429_1605 (size=12949) 2024-12-06T14:44:34,137 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/a85440c018bc4066802523e576b01177 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a85440c018bc4066802523e576b01177 2024-12-06T14:44:34,138 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/177f635aa1da4f6b9ebae6c50583e0eb 2024-12-06T14:44:34,140 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into a85440c018bc4066802523e576b01177(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:34,140 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:34,140 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496273667; duration=0sec 2024-12-06T14:44:34,140 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:34,141 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:34,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/e82f8d929dd642b8a2bcdf8c751d3807 is 50, key is test_row_0/B:col10/1733496272829/Put/seqid=0 2024-12-06T14:44:34,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742430_1606 (size=12301) 2024-12-06T14:44:34,145 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/e82f8d929dd642b8a2bcdf8c751d3807 2024-12-06T14:44:34,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/ac367bd956d748c38b60c53c1adf2f9a is 50, key is test_row_0/C:col10/1733496272829/Put/seqid=0 2024-12-06T14:44:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742431_1607 (size=12301) 2024-12-06T14:44:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496334168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T14:44:34,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496334378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:34,522 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/9110842258964f53a0d95da554e13fc6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/9110842258964f53a0d95da554e13fc6 2024-12-06T14:44:34,525 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into 9110842258964f53a0d95da554e13fc6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:34,525 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:34,525 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496273667; duration=0sec 2024-12-06T14:44:34,525 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:34,525 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:34,554 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/ac367bd956d748c38b60c53c1adf2f9a 2024-12-06T14:44:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/177f635aa1da4f6b9ebae6c50583e0eb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb 2024-12-06T14:44:34,559 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb, entries=150, sequenceid=300, filesize=30.5 K 2024-12-06T14:44:34,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/e82f8d929dd642b8a2bcdf8c751d3807 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e82f8d929dd642b8a2bcdf8c751d3807 2024-12-06T14:44:34,563 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e82f8d929dd642b8a2bcdf8c751d3807, entries=150, sequenceid=300, filesize=12.0 K 2024-12-06T14:44:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/ac367bd956d748c38b60c53c1adf2f9a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac367bd956d748c38b60c53c1adf2f9a 2024-12-06T14:44:34,567 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac367bd956d748c38b60c53c1adf2f9a, entries=150, sequenceid=300, filesize=12.0 K 2024-12-06T14:44:34,567 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 3623e1fd493fe297ada6ccd9ac6123ae in 889ms, sequenceid=300, compaction requested=false 2024-12-06T14:44:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-06T14:44:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-06T14:44:34,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-06T14:44:34,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3480 sec 2024-12-06T14:44:34,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.3510 sec 2024-12-06T14:44:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:34,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T14:44:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:34,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a7804ce35d18470899eb5d63720ca0c6_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:34,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742432_1608 (size=12454) 2024-12-06T14:44:34,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496334723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496334828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:35,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496335038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:35,099 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:35,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a7804ce35d18470899eb5d63720ca0c6_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a7804ce35d18470899eb5d63720ca0c6_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:35,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b59cff4df973474f933402d1b2c00c58, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:35,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b59cff4df973474f933402d1b2c00c58 is 175, key is test_row_0/A:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:35,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742433_1609 (size=31255) 2024-12-06T14:44:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496335348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T14:44:35,369 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:35,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-06T14:44:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T14:44:35,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:35,371 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:35,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T14:44:35,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b59cff4df973474f933402d1b2c00c58 2024-12-06T14:44:35,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/6a9eea2afb2744699425818e883ab505 is 50, key is test_row_0/B:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:35,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742434_1610 (size=12301) 2024-12-06T14:44:35,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:35,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T14:44:35,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496335858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:35,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/6a9eea2afb2744699425818e883ab505 2024-12-06T14:44:35,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/7ebe6c265d3a471d9f81dd7efbb38f65 is 50, key is test_row_0/C:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:35,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742435_1611 (size=12301) 2024-12-06T14:44:35,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:35,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:35,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:35,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T14:44:36,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:36,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:36,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:36,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:36,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60328 deadline: 1733496336228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:36,230 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:36,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60296 deadline: 1733496336248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:36,252 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_1' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:36,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:36,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:36,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:36,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:36,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/7ebe6c265d3a471d9f81dd7efbb38f65 2024-12-06T14:44:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b59cff4df973474f933402d1b2c00c58 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58 2024-12-06T14:44:36,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58, entries=150, sequenceid=321, filesize=30.5 K 2024-12-06T14:44:36,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/6a9eea2afb2744699425818e883ab505 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/6a9eea2afb2744699425818e883ab505 2024-12-06T14:44:36,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/6a9eea2afb2744699425818e883ab505, entries=150, sequenceid=321, filesize=12.0 K 2024-12-06T14:44:36,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/7ebe6c265d3a471d9f81dd7efbb38f65 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/7ebe6c265d3a471d9f81dd7efbb38f65 2024-12-06T14:44:36,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/7ebe6c265d3a471d9f81dd7efbb38f65, entries=150, sequenceid=321, filesize=12.0 K 2024-12-06T14:44:36,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3623e1fd493fe297ada6ccd9ac6123ae in 1646ms, sequenceid=321, compaction requested=true 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:36,336 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:36,336 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:36,336 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:36,336 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:36,336 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:36,336 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:36,336 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,336 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,337 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/b160a295132546e3a772507030f27dff, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e82f8d929dd642b8a2bcdf8c751d3807, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/6a9eea2afb2744699425818e883ab505] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.7 K 2024-12-06T14:44:36,337 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a85440c018bc4066802523e576b01177, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=92.2 K 2024-12-06T14:44:36,337 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a85440c018bc4066802523e576b01177, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58] 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting b160a295132546e3a772507030f27dff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733496271644 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85440c018bc4066802523e576b01177, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733496271644 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting e82f8d929dd642b8a2bcdf8c751d3807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733496272829 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a9eea2afb2744699425818e883ab505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496274017 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 177f635aa1da4f6b9ebae6c50583e0eb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733496272829 2024-12-06T14:44:36,337 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59cff4df973474f933402d1b2c00c58, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496274017 2024-12-06T14:44:36,341 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:36,343 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:36,343 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/8a6ac3726dfe46d4962e2d326e1799ac is 50, key is test_row_0/B:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:36,349 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206a9e4d353d5a641c2b678b0527d84c544_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:36,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742436_1612 (size=13051) 2024-12-06T14:44:36,351 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206a9e4d353d5a641c2b678b0527d84c544_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:36,351 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a9e4d353d5a641c2b678b0527d84c544_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:36,356 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/8a6ac3726dfe46d4962e2d326e1799ac as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/8a6ac3726dfe46d4962e2d326e1799ac 2024-12-06T14:44:36,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742437_1613 (size=4469) 2024-12-06T14:44:36,362 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into 8a6ac3726dfe46d4962e2d326e1799ac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:36,362 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:36,362 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496276336; duration=0sec 2024-12-06T14:44:36,362 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:36,362 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:36,362 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:36,363 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:36,363 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:36,363 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,363 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/9110842258964f53a0d95da554e13fc6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac367bd956d748c38b60c53c1adf2f9a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/7ebe6c265d3a471d9f81dd7efbb38f65] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.7 K 2024-12-06T14:44:36,363 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 9110842258964f53a0d95da554e13fc6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733496271644 2024-12-06T14:44:36,363 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting ac367bd956d748c38b60c53c1adf2f9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733496272829 2024-12-06T14:44:36,364 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ebe6c265d3a471d9f81dd7efbb38f65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496274017 2024-12-06T14:44:36,369 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:36,369 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6779f75185e94277bb18468e2be35bcb is 50, key is test_row_0/C:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:36,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742438_1614 (size=13051) 2024-12-06T14:44:36,376 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/6779f75185e94277bb18468e2be35bcb as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6779f75185e94277bb18468e2be35bcb 2024-12-06T14:44:36,380 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into 6779f75185e94277bb18468e2be35bcb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:36,380 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:36,380 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496276336; duration=0sec 2024-12-06T14:44:36,381 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:36,381 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:36,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T14:44:36,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:36,438 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T14:44:36,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:36,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206202b0a79e27145e9a205b93df812b8d0_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496274707/Put/seqid=0 2024-12-06T14:44:36,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742439_1615 (size=12454) 2024-12-06T14:44:36,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:36,476 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206202b0a79e27145e9a205b93df812b8d0_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206202b0a79e27145e9a205b93df812b8d0_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/88f3e79f7b174fb58f1d14b3b099710f, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:36,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/88f3e79f7b174fb58f1d14b3b099710f is 175, key is test_row_0/A:col10/1733496274707/Put/seqid=0 2024-12-06T14:44:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T14:44:36,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742440_1616 (size=31255) 2024-12-06T14:44:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:36,762 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:36,763 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/1c8f1c2f122b4a41b801c8298c9a11bd is 175, key is test_row_0/A:col10/1733496274017/Put/seqid=0 2024-12-06T14:44:36,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742441_1617 (size=32005) 2024-12-06T14:44:36,827 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/1c8f1c2f122b4a41b801c8298c9a11bd as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1c8f1c2f122b4a41b801c8298c9a11bd 2024-12-06T14:44:36,834 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into 1c8f1c2f122b4a41b801c8298c9a11bd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:36,834 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:36,834 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496276336; duration=0sec 2024-12-06T14:44:36,834 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:36,834 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496336868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:36,881 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:36,919 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/88f3e79f7b174fb58f1d14b3b099710f 2024-12-06T14:44:36,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496336873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496336881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/c3a1633d5ad945baa1ab52da3007497a is 50, key is test_row_0/B:col10/1733496274707/Put/seqid=0 2024-12-06T14:44:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:36,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742442_1618 (size=12301) 2024-12-06T14:44:37,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496337049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496337070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496337268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496337279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,370 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/c3a1633d5ad945baa1ab52da3007497a 2024-12-06T14:44:37,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/20b701dc1c7349b8adbeb31765d2efd6 is 50, key is test_row_0/C:col10/1733496274707/Put/seqid=0 2024-12-06T14:44:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742443_1619 (size=12301) 2024-12-06T14:44:37,409 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/20b701dc1c7349b8adbeb31765d2efd6 2024-12-06T14:44:37,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/88f3e79f7b174fb58f1d14b3b099710f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f 2024-12-06T14:44:37,434 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f, entries=150, sequenceid=337, filesize=30.5 K 2024-12-06T14:44:37,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/c3a1633d5ad945baa1ab52da3007497a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c3a1633d5ad945baa1ab52da3007497a 2024-12-06T14:44:37,440 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c3a1633d5ad945baa1ab52da3007497a, entries=150, sequenceid=337, filesize=12.0 K 2024-12-06T14:44:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/20b701dc1c7349b8adbeb31765d2efd6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/20b701dc1c7349b8adbeb31765d2efd6 2024-12-06T14:44:37,469 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/20b701dc1c7349b8adbeb31765d2efd6, entries=150, sequenceid=337, filesize=12.0 K 2024-12-06T14:44:37,475 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3623e1fd493fe297ada6ccd9ac6123ae in 1037ms, sequenceid=337, compaction requested=false 2024-12-06T14:44:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:37,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:37,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-06T14:44:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-06T14:44:37,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-06T14:44:37,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1090 sec 2024-12-06T14:44:37,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1140 sec 2024-12-06T14:44:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T14:44:37,519 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:37,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T14:44:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-06T14:44:37,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T14:44:37,554 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T14:44:37,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T14:44:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:37,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:44:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:37,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c64abc4796224ba7b30ba21ac1286b27_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:37,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742444_1620 (size=14994) 2024-12-06T14:44:37,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:37,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c64abc4796224ba7b30ba21ac1286b27_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c64abc4796224ba7b30ba21ac1286b27_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:37,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/dde127c4c288427b8c771b46776518ad, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:37,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/dde127c4c288427b8c771b46776518ad is 175, key is test_row_0/A:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742445_1621 (size=39949) 2024-12-06T14:44:37,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/dde127c4c288427b8c771b46776518ad 2024-12-06T14:44:37,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/db28c51c9aa648199639b00ca10c2b9d is 50, key is test_row_0/B:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:37,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742446_1622 (size=12301) 2024-12-06T14:44:37,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496337684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496337684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T14:44:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:37,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:37,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496337798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496337809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:37,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T14:44:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:37,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496338009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T14:44:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:44:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496338019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/db28c51c9aa648199639b00ca10c2b9d 2024-12-06T14:44:38,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf0665df62c94d73861678cd705aff5f is 50, key is test_row_0/C:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:38,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742447_1623 (size=12301) 2024-12-06T14:44:38,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf0665df62c94d73861678cd705aff5f 2024-12-06T14:44:38,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/dde127c4c288427b8c771b46776518ad as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad 2024-12-06T14:44:38,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad, entries=200, sequenceid=357, filesize=39.0 K 2024-12-06T14:44:38,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/db28c51c9aa648199639b00ca10c2b9d as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/db28c51c9aa648199639b00ca10c2b9d 2024-12-06T14:44:38,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/db28c51c9aa648199639b00ca10c2b9d, entries=150, sequenceid=357, filesize=12.0 K 2024-12-06T14:44:38,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/cf0665df62c94d73861678cd705aff5f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf0665df62c94d73861678cd705aff5f 2024-12-06T14:44:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:38,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf0665df62c94d73861678cd705aff5f, entries=150, sequenceid=357, filesize=12.0 K 2024-12-06T14:44:38,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 3623e1fd493fe297ada6ccd9ac6123ae in 598ms, sequenceid=357, compaction requested=true 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:38,180 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:38,180 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:38,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35831 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,182 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:38,183 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:38,183 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:38,183 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:38,183 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:38,183 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,183 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,183 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/8a6ac3726dfe46d4962e2d326e1799ac, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c3a1633d5ad945baa1ab52da3007497a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/db28c51c9aa648199639b00ca10c2b9d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.8 K 2024-12-06T14:44:38,183 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1c8f1c2f122b4a41b801c8298c9a11bd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=100.8 K 2024-12-06T14:44:38,184 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,184 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1c8f1c2f122b4a41b801c8298c9a11bd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad] 2024-12-06T14:44:38,184 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6ac3726dfe46d4962e2d326e1799ac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496274017 2024-12-06T14:44:38,184 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8f1c2f122b4a41b801c8298c9a11bd, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496274017 2024-12-06T14:44:38,184 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a1633d5ad945baa1ab52da3007497a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733496274707 2024-12-06T14:44:38,184 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f3e79f7b174fb58f1d14b3b099710f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733496274707 2024-12-06T14:44:38,186 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting db28c51c9aa648199639b00ca10c2b9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733496276871 2024-12-06T14:44:38,186 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde127c4c288427b8c771b46776518ad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733496276864 2024-12-06T14:44:38,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f434195316a74f269a823cb26f948a70_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496277678/Put/seqid=0 2024-12-06T14:44:38,194 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#523 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:38,194 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/a5ba742b83c646238f46e756ed942c4a is 50, key is test_row_0/B:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:38,196 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:38,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742448_1624 (size=12454) 2024-12-06T14:44:38,225 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206a9df2170b6b1490f925fe9d63038a862_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:38,226 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206a9df2170b6b1490f925fe9d63038a862_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:38,226 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a9df2170b6b1490f925fe9d63038a862_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:38,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742449_1625 (size=13153) 2024-12-06T14:44:38,232 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/a5ba742b83c646238f46e756ed942c4a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a5ba742b83c646238f46e756ed942c4a 2024-12-06T14:44:38,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742450_1626 (size=4469) 2024-12-06T14:44:38,236 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#524 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:38,236 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/57d0d34e029944a287076de55e2a4e46 is 175, key is test_row_0/A:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:38,238 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into a5ba742b83c646238f46e756ed942c4a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:38,238 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:38,238 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496278180; duration=0sec 2024-12-06T14:44:38,238 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:38,238 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:38,238 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:38,239 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:38,239 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:38,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742451_1627 (size=32107) 2024-12-06T14:44:38,240 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:38,240 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6779f75185e94277bb18468e2be35bcb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/20b701dc1c7349b8adbeb31765d2efd6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf0665df62c94d73861678cd705aff5f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.8 K 2024-12-06T14:44:38,240 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 6779f75185e94277bb18468e2be35bcb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733496274017 2024-12-06T14:44:38,240 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b701dc1c7349b8adbeb31765d2efd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733496274707 2024-12-06T14:44:38,241 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0665df62c94d73861678cd705aff5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733496276871 2024-12-06T14:44:38,245 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/57d0d34e029944a287076de55e2a4e46 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/57d0d34e029944a287076de55e2a4e46 2024-12-06T14:44:38,250 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into 57d0d34e029944a287076de55e2a4e46(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:38,250 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:38,250 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496278180; duration=0sec 2024-12-06T14:44:38,250 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:38,250 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:38,253 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:38,254 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/12e3fc2cec8e401782fb61df69ad174a is 50, key is test_row_0/C:col10/1733496276871/Put/seqid=0 2024-12-06T14:44:38,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742452_1628 (size=13153) 2024-12-06T14:44:38,275 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/12e3fc2cec8e401782fb61df69ad174a as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/12e3fc2cec8e401782fb61df69ad174a 2024-12-06T14:44:38,278 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into 12e3fc2cec8e401782fb61df69ad174a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:38,278 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:38,278 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496278180; duration=0sec 2024-12-06T14:44:38,278 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:38,278 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. as already flushing 2024-12-06T14:44:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496338362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496338368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,419 INFO [Thread-2351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2351. 2024-12-06T14:44:38,419 DEBUG [Thread-2351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:38,419 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,419 INFO [Thread-2357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2357. 2024-12-06T14:44:38,419 DEBUG [Thread-2357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:38,419 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,419 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:38,419 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:38,420 INFO [Thread-2355 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2355. 2024-12-06T14:44:38,420 DEBUG [Thread-2355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:38,420 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,420 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:38,421 INFO [Thread-2349 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2349. 2024-12-06T14:44:38,421 DEBUG [Thread-2349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:38,421 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,421 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:38,421 INFO [Thread-2353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2353. 2024-12-06T14:44:38,421 DEBUG [Thread-2353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:38,421 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,421 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:38,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496338469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,547 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T14:44:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:38,610 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f434195316a74f269a823cb26f948a70_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f434195316a74f269a823cb26f948a70_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:38,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de2dad581e644c9aa94ab01e3a00186b, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de2dad581e644c9aa94ab01e3a00186b is 175, key is test_row_0/A:col10/1733496277678/Put/seqid=0 2024-12-06T14:44:38,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742453_1629 (size=31255) 2024-12-06T14:44:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496338679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733496338878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60280 deadline: 1733496338900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,901 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bb7ea3e46d9c,35831,1733496100253 for 'test_row_0' in 3623e1fd493fe297ada6ccd9ac6123ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T14:44:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:38,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496338989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:39,016 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de2dad581e644c9aa94ab01e3a00186b 2024-12-06T14:44:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/7a6389fa8057472bb8788a32d0bd6615 is 50, key is test_row_0/B:col10/1733496277678/Put/seqid=0 2024-12-06T14:44:39,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742454_1630 (size=12301) 2024-12-06T14:44:39,427 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/7a6389fa8057472bb8788a32d0bd6615 2024-12-06T14:44:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/012c96eab62645b2a54955ae3783d745 is 50, key is test_row_0/C:col10/1733496277678/Put/seqid=0 2024-12-06T14:44:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742455_1631 (size=12301) 2024-12-06T14:44:39,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:44:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1733496339498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 , the old value is region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T14:44:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., hostname=bb7ea3e46d9c,35831,1733496100253, seqNum=5 because the exception is null or not the one we care about 2024-12-06T14:44:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:39,851 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/012c96eab62645b2a54955ae3783d745 2024-12-06T14:44:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/de2dad581e644c9aa94ab01e3a00186b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b 2024-12-06T14:44:39,857 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b, entries=150, sequenceid=371, filesize=30.5 K 2024-12-06T14:44:39,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/7a6389fa8057472bb8788a32d0bd6615 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a6389fa8057472bb8788a32d0bd6615 2024-12-06T14:44:39,860 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a6389fa8057472bb8788a32d0bd6615, entries=150, sequenceid=371, filesize=12.0 K 2024-12-06T14:44:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/012c96eab62645b2a54955ae3783d745 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/012c96eab62645b2a54955ae3783d745 2024-12-06T14:44:39,863 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/012c96eab62645b2a54955ae3783d745, entries=150, sequenceid=371, filesize=12.0 K 2024-12-06T14:44:39,863 INFO [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3623e1fd493fe297ada6ccd9ac6123ae in 1681ms, sequenceid=371, compaction requested=false 2024-12-06T14:44:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-06T14:44:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-06T14:44:39,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-06T14:44:39,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3110 sec 2024-12-06T14:44:39,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3460 sec 2024-12-06T14:44:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35831 {}] regionserver.HRegion(8855): Flush requested on 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:39,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T14:44:39,890 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-12-06T14:44:39,890 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:39,890 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:39,890 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:39,890 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:39,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061b18d7589fec4dc981b5cb830ca6e010_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:39,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742456_1632 (size=12454) 2024-12-06T14:44:40,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:40,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061b18d7589fec4dc981b5cb830ca6e010_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061b18d7589fec4dc981b5cb830ca6e010_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:40,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2cf7a07208c24b05a859cd08b5fc26bc, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:40,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2cf7a07208c24b05a859cd08b5fc26bc is 175, key is test_row_0/A:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:40,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742457_1633 (size=31255) 2024-12-06T14:44:40,509 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-12-06T14:44:40,509 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:40,509 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:40,510 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:40,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:40,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2cf7a07208c24b05a859cd08b5fc26bc 2024-12-06T14:44:40,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/baa24a1a07574f8f9fa28d69e9acb7f2 is 50, key is test_row_0/B:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:40,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742458_1634 (size=12301) 2024-12-06T14:44:41,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/baa24a1a07574f8f9fa28d69e9acb7f2 2024-12-06T14:44:41,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/d35bb730f4454871905e98fc119af185 is 50, key is test_row_0/C:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:41,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742459_1635 (size=12301) 2024-12-06T14:44:41,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/d35bb730f4454871905e98fc119af185 2024-12-06T14:44:41,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/2cf7a07208c24b05a859cd08b5fc26bc as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc 2024-12-06T14:44:41,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc, entries=150, sequenceid=393, filesize=30.5 K 2024-12-06T14:44:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/baa24a1a07574f8f9fa28d69e9acb7f2 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/baa24a1a07574f8f9fa28d69e9acb7f2 2024-12-06T14:44:41,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/baa24a1a07574f8f9fa28d69e9acb7f2, entries=150, sequenceid=393, filesize=12.0 K 2024-12-06T14:44:41,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/d35bb730f4454871905e98fc119af185 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d35bb730f4454871905e98fc119af185 2024-12-06T14:44:41,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d35bb730f4454871905e98fc119af185, entries=150, sequenceid=393, filesize=12.0 K 2024-12-06T14:44:41,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 3623e1fd493fe297ada6ccd9ac6123ae in 1657ms, sequenceid=393, compaction requested=true 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T14:44:41,546 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3623e1fd493fe297ada6ccd9ac6123ae:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T14:44:41,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:41,546 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/A is initiating minor compaction (all files) 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/B is initiating minor compaction (all files) 2024-12-06T14:44:41,547 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/A in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:41,547 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/B in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:41,547 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/57d0d34e029944a287076de55e2a4e46, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=92.4 K 2024-12-06T14:44:41,547 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a5ba742b83c646238f46e756ed942c4a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a6389fa8057472bb8788a32d0bd6615, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/baa24a1a07574f8f9fa28d69e9acb7f2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.9 K 2024-12-06T14:44:41,547 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. files: [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/57d0d34e029944a287076de55e2a4e46, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc] 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5ba742b83c646238f46e756ed942c4a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733496276871 2024-12-06T14:44:41,547 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d0d34e029944a287076de55e2a4e46, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733496276871 2024-12-06T14:44:41,548 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6389fa8057472bb8788a32d0bd6615, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733496277673 2024-12-06T14:44:41,548 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting de2dad581e644c9aa94ab01e3a00186b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733496277673 2024-12-06T14:44:41,548 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa24a1a07574f8f9fa28d69e9acb7f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733496278358 2024-12-06T14:44:41,548 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf7a07208c24b05a859cd08b5fc26bc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733496278358 2024-12-06T14:44:41,553 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:41,554 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:41,554 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/4ecdd1ddf04e423789bd3ece5ef8a36b is 50, key is test_row_0/B:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:41,555 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206972c4d91ad084b4b99e9bb2be56b9b54_3623e1fd493fe297ada6ccd9ac6123ae store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:41,556 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206972c4d91ad084b4b99e9bb2be56b9b54_3623e1fd493fe297ada6ccd9ac6123ae, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:41,556 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206972c4d91ad084b4b99e9bb2be56b9b54_3623e1fd493fe297ada6ccd9ac6123ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742460_1636 (size=13255) 2024-12-06T14:44:41,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742461_1637 (size=4469) 2024-12-06T14:44:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T14:44:41,719 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:41,964 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:41,966 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/c5a66b27d15642b983ede4d49eb5a684 is 175, key is test_row_0/A:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:41,968 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/4ecdd1ddf04e423789bd3ece5ef8a36b as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4ecdd1ddf04e423789bd3ece5ef8a36b 2024-12-06T14:44:41,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742462_1638 (size=32209) 2024-12-06T14:44:41,972 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/B of 3623e1fd493fe297ada6ccd9ac6123ae into 4ecdd1ddf04e423789bd3ece5ef8a36b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:41,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:41,973 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/B, priority=13, startTime=1733496281546; duration=0sec 2024-12-06T14:44:41,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T14:44:41,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:B 2024-12-06T14:44:41,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T14:44:41,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T14:44:41,973 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1541): 3623e1fd493fe297ada6ccd9ac6123ae/C is initiating minor compaction (all files) 2024-12-06T14:44:41,974 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3623e1fd493fe297ada6ccd9ac6123ae/C in TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:41,974 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/12e3fc2cec8e401782fb61df69ad174a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/012c96eab62645b2a54955ae3783d745, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d35bb730f4454871905e98fc119af185] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp, totalSize=36.9 K 2024-12-06T14:44:41,974 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e3fc2cec8e401782fb61df69ad174a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733496276871 2024-12-06T14:44:41,974 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 012c96eab62645b2a54955ae3783d745, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733496277673 2024-12-06T14:44:41,974 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d35bb730f4454871905e98fc119af185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733496278358 2024-12-06T14:44:41,980 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3623e1fd493fe297ada6ccd9ac6123ae#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T14:44:41,981 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/3ca6e7fe703c453580ee8a4bee40c790 is 50, key is test_row_0/C:col10/1733496278358/Put/seqid=0 2024-12-06T14:44:41,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742463_1639 (size=13255) 2024-12-06T14:44:42,382 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/c5a66b27d15642b983ede4d49eb5a684 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/c5a66b27d15642b983ede4d49eb5a684 2024-12-06T14:44:42,386 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/A of 3623e1fd493fe297ada6ccd9ac6123ae into c5a66b27d15642b983ede4d49eb5a684(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:42,386 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:42,386 INFO [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/A, priority=13, startTime=1733496281546; duration=0sec 2024-12-06T14:44:42,386 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:42,386 DEBUG [RS:0;bb7ea3e46d9c:35831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:A 2024-12-06T14:44:42,388 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/3ca6e7fe703c453580ee8a4bee40c790 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3ca6e7fe703c453580ee8a4bee40c790 2024-12-06T14:44:42,392 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3623e1fd493fe297ada6ccd9ac6123ae/C of 3623e1fd493fe297ada6ccd9ac6123ae into 3ca6e7fe703c453580ee8a4bee40c790(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T14:44:42,392 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3623e1fd493fe297ada6ccd9ac6123ae: 2024-12-06T14:44:42,392 INFO [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., storeName=3623e1fd493fe297ada6ccd9ac6123ae/C, priority=13, startTime=1733496281546; duration=0sec 2024-12-06T14:44:42,392 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T14:44:42,392 DEBUG [RS:0;bb7ea3e46d9c:35831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3623e1fd493fe297ada6ccd9ac6123ae:C 2024-12-06T14:44:42,943 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-12-06T14:44:42,943 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:42,943 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:42,943 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:42,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:46,319 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-12-06T14:44:46,319 INFO [Thread-2338 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2338. 2024-12-06T14:44:46,319 DEBUG [Thread-2338 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:46,319 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T14:44:46,320 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:46,320 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:46,320 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:46,320 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:46,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:46,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3691 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3553 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3601 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3697 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3579 2024-12-06T14:44:46,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T14:44:46,320 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:44:46,321 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:44:46,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:46,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:46,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:46,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T14:44:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T14:44:46,325 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496286325"}]},"ts":"1733496286325"} 2024-12-06T14:44:46,326 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T14:44:46,326 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T14:44:46,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T14:44:46,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, UNASSIGN}] 2024-12-06T14:44:46,329 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, UNASSIGN 2024-12-06T14:44:46,329 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=CLOSING, regionLocation=bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:46,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, UNASSIGN because future has completed 2024-12-06T14:44:46,330 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T14:44:46,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253}] 2024-12-06T14:44:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T14:44:46,482 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 3623e1fd493fe297ada6ccd9ac6123ae, disabling compactions & flushes 2024-12-06T14:44:46,483 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. after waiting 0 ms 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:46,483 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 3623e1fd493fe297ada6ccd9ac6123ae 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=A 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=B 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3623e1fd493fe297ada6ccd9ac6123ae, store=C 2024-12-06T14:44:46,483 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T14:44:46,489 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069914105a2bcb414fac25f590419fdf4b_3623e1fd493fe297ada6ccd9ac6123ae is 50, key is test_row_0/A:col10/1733496286318/Put/seqid=0 2024-12-06T14:44:46,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742464_1640 (size=12454) 2024-12-06T14:44:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T14:44:46,894 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:44:46,903 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069914105a2bcb414fac25f590419fdf4b_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069914105a2bcb414fac25f590419fdf4b_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:46,905 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b96431753b0c43c2a2598fab11425b47, store: [table=TestAcidGuarantees family=A region=3623e1fd493fe297ada6ccd9ac6123ae] 2024-12-06T14:44:46,905 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b96431753b0c43c2a2598fab11425b47 is 175, key is test_row_0/A:col10/1733496286318/Put/seqid=0 2024-12-06T14:44:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742465_1641 (size=31255) 2024-12-06T14:44:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T14:44:47,313 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b96431753b0c43c2a2598fab11425b47 2024-12-06T14:44:47,326 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9db25e9eaf94498f836750b27b726e52 is 50, key is test_row_0/B:col10/1733496286318/Put/seqid=0 2024-12-06T14:44:47,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742466_1642 (size=12301) 2024-12-06T14:44:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T14:44:47,731 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9db25e9eaf94498f836750b27b726e52 2024-12-06T14:44:47,744 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/55dbcdf114c9441f9d6cbf832f03fcb1 is 50, key is test_row_0/C:col10/1733496286318/Put/seqid=0 2024-12-06T14:44:47,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742467_1643 (size=12301) 2024-12-06T14:44:48,150 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/55dbcdf114c9441f9d6cbf832f03fcb1 2024-12-06T14:44:48,160 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/A/b96431753b0c43c2a2598fab11425b47 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b96431753b0c43c2a2598fab11425b47 2024-12-06T14:44:48,164 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b96431753b0c43c2a2598fab11425b47, entries=150, sequenceid=403, filesize=30.5 K 2024-12-06T14:44:48,164 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/B/9db25e9eaf94498f836750b27b726e52 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9db25e9eaf94498f836750b27b726e52 2024-12-06T14:44:48,168 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9db25e9eaf94498f836750b27b726e52, entries=150, sequenceid=403, filesize=12.0 K 2024-12-06T14:44:48,168 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/.tmp/C/55dbcdf114c9441f9d6cbf832f03fcb1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/55dbcdf114c9441f9d6cbf832f03fcb1 2024-12-06T14:44:48,172 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/55dbcdf114c9441f9d6cbf832f03fcb1, entries=150, sequenceid=403, filesize=12.0 K 2024-12-06T14:44:48,172 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3623e1fd493fe297ada6ccd9ac6123ae in 1689ms, sequenceid=403, compaction requested=false 2024-12-06T14:44:48,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/0f6e285f40d54b43b3e2835a95bde772, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/6e684d50b17a4071907e73f99be2842c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b48567c80ac4416096155293c4ef81c7, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1ed8e281f7d74eaa9cb24d9c93c54011, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8a3261ffbc6141c689e15db0448e16e5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a85440c018bc4066802523e576b01177, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1c8f1c2f122b4a41b801c8298c9a11bd, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/57d0d34e029944a287076de55e2a4e46, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc] to archive 2024-12-06T14:44:48,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:44:48,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/3b41905d46d14634937ee686122bc9f3 2024-12-06T14:44:48,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2bac29a042964f2c9c884b29c1d15e9f 2024-12-06T14:44:48,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/394bd7d77e684d94a4bbc08d3782ea90 2024-12-06T14:44:48,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/0f6e285f40d54b43b3e2835a95bde772 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/0f6e285f40d54b43b3e2835a95bde772 2024-12-06T14:44:48,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a0d144af582340018035e24c59965da8 2024-12-06T14:44:48,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/456b45b293034171827256d2a5691835 2024-12-06T14:44:48,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/6e684d50b17a4071907e73f99be2842c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/6e684d50b17a4071907e73f99be2842c 2024-12-06T14:44:48,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8c5afa761e344d899d9935902f0efa08 2024-12-06T14:44:48,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/02c842e3ba7241d58bf53324e1a03e0d 2024-12-06T14:44:48,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de9919b4388f48c1a8cc1d0926515372 2024-12-06T14:44:48,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b48567c80ac4416096155293c4ef81c7 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b48567c80ac4416096155293c4ef81c7 2024-12-06T14:44:48,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/41dd0044f3374abfb76ae75d52fa47c6 2024-12-06T14:44:48,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de3b784beae840fab46b7c54c6ee315c 2024-12-06T14:44:48,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1ed8e281f7d74eaa9cb24d9c93c54011 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1ed8e281f7d74eaa9cb24d9c93c54011 2024-12-06T14:44:48,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a95c06714c634ba7b36ada45a2c2eea1 2024-12-06T14:44:48,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/65efdb56801543fe9ec89d6f3aca4ff9 2024-12-06T14:44:48,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8a3261ffbc6141c689e15db0448e16e5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/8a3261ffbc6141c689e15db0448e16e5 2024-12-06T14:44:48,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/51a6241fbc9f45c08f2d970ca6c1cf54 2024-12-06T14:44:48,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a7afe75aa7474219b97dafe04b98cae5 2024-12-06T14:44:48,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a85440c018bc4066802523e576b01177 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/a85440c018bc4066802523e576b01177 2024-12-06T14:44:48,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/177f635aa1da4f6b9ebae6c50583e0eb 2024-12-06T14:44:48,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1c8f1c2f122b4a41b801c8298c9a11bd to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/1c8f1c2f122b4a41b801c8298c9a11bd 2024-12-06T14:44:48,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b59cff4df973474f933402d1b2c00c58 2024-12-06T14:44:48,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/88f3e79f7b174fb58f1d14b3b099710f 2024-12-06T14:44:48,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/dde127c4c288427b8c771b46776518ad 2024-12-06T14:44:48,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/57d0d34e029944a287076de55e2a4e46 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/57d0d34e029944a287076de55e2a4e46 2024-12-06T14:44:48,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/de2dad581e644c9aa94ab01e3a00186b 2024-12-06T14:44:48,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/2cf7a07208c24b05a859cd08b5fc26bc 2024-12-06T14:44:48,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a54ed0b6a734a82bb5bd43cbb687890, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c5d52ee527094986808a8d2881d712f6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9828366e7dd24795a1e4942790f0edd8, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9b583a18987a4cf0b8d1e8c2eddd8abb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/0b37c2aff8c944e29bf3ce7598b4ff00, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e76b2c96b8424a33a64d23d32ee80a53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ed8a1996fb2145be94df5de88a96a0d0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/be3925b830a94f1f893d0630d1d49373, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ddcb0d5279e444ad9766ecf69a69a988, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/fbbdf528df924391a6a91b41c3dc2070, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/f72eae70d41249f994d823e656a7f199, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/d230783ee625471a943817e863b36ab3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ce6bb0619bd04d9fa9ac53da190c6c06, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/da4bd412358c48429e3d0c0a41dd8cd3, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/46fe172181cc4b4a9d22eaf5cdab4408, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/cd4ff6dde8b2450b86414cee2e09d36e, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4e4a1ec5b5e94b84992a9eb54ddec5dc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a02b2cccb79846fd90136cc15f8df561, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/b160a295132546e3a772507030f27dff, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/bb1988ef526447139d2c54564030b6d9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e82f8d929dd642b8a2bcdf8c751d3807, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/8a6ac3726dfe46d4962e2d326e1799ac, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/6a9eea2afb2744699425818e883ab505, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c3a1633d5ad945baa1ab52da3007497a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a5ba742b83c646238f46e756ed942c4a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/db28c51c9aa648199639b00ca10c2b9d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a6389fa8057472bb8788a32d0bd6615, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/baa24a1a07574f8f9fa28d69e9acb7f2] to archive 2024-12-06T14:44:48,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:44:48,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a54ed0b6a734a82bb5bd43cbb687890 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a54ed0b6a734a82bb5bd43cbb687890 2024-12-06T14:44:48,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c5d52ee527094986808a8d2881d712f6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c5d52ee527094986808a8d2881d712f6 2024-12-06T14:44:48,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9828366e7dd24795a1e4942790f0edd8 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9828366e7dd24795a1e4942790f0edd8 2024-12-06T14:44:48,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9b583a18987a4cf0b8d1e8c2eddd8abb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9b583a18987a4cf0b8d1e8c2eddd8abb 2024-12-06T14:44:48,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/0b37c2aff8c944e29bf3ce7598b4ff00 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/0b37c2aff8c944e29bf3ce7598b4ff00 2024-12-06T14:44:48,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e76b2c96b8424a33a64d23d32ee80a53 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e76b2c96b8424a33a64d23d32ee80a53 2024-12-06T14:44:48,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ed8a1996fb2145be94df5de88a96a0d0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ed8a1996fb2145be94df5de88a96a0d0 2024-12-06T14:44:48,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/be3925b830a94f1f893d0630d1d49373 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/be3925b830a94f1f893d0630d1d49373 2024-12-06T14:44:48,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ddcb0d5279e444ad9766ecf69a69a988 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ddcb0d5279e444ad9766ecf69a69a988 2024-12-06T14:44:48,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/fbbdf528df924391a6a91b41c3dc2070 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/fbbdf528df924391a6a91b41c3dc2070 2024-12-06T14:44:48,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/f72eae70d41249f994d823e656a7f199 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/f72eae70d41249f994d823e656a7f199 2024-12-06T14:44:48,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/d230783ee625471a943817e863b36ab3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/d230783ee625471a943817e863b36ab3 2024-12-06T14:44:48,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ce6bb0619bd04d9fa9ac53da190c6c06 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/ce6bb0619bd04d9fa9ac53da190c6c06 2024-12-06T14:44:48,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/da4bd412358c48429e3d0c0a41dd8cd3 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/da4bd412358c48429e3d0c0a41dd8cd3 2024-12-06T14:44:48,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/46fe172181cc4b4a9d22eaf5cdab4408 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/46fe172181cc4b4a9d22eaf5cdab4408 2024-12-06T14:44:48,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/cd4ff6dde8b2450b86414cee2e09d36e to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/cd4ff6dde8b2450b86414cee2e09d36e 2024-12-06T14:44:48,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4e4a1ec5b5e94b84992a9eb54ddec5dc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4e4a1ec5b5e94b84992a9eb54ddec5dc 2024-12-06T14:44:48,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a02b2cccb79846fd90136cc15f8df561 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a02b2cccb79846fd90136cc15f8df561 2024-12-06T14:44:48,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/b160a295132546e3a772507030f27dff to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/b160a295132546e3a772507030f27dff 2024-12-06T14:44:48,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/bb1988ef526447139d2c54564030b6d9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/bb1988ef526447139d2c54564030b6d9 2024-12-06T14:44:48,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e82f8d929dd642b8a2bcdf8c751d3807 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/e82f8d929dd642b8a2bcdf8c751d3807 2024-12-06T14:44:48,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/8a6ac3726dfe46d4962e2d326e1799ac to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/8a6ac3726dfe46d4962e2d326e1799ac 2024-12-06T14:44:48,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/6a9eea2afb2744699425818e883ab505 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/6a9eea2afb2744699425818e883ab505 2024-12-06T14:44:48,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c3a1633d5ad945baa1ab52da3007497a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/c3a1633d5ad945baa1ab52da3007497a 2024-12-06T14:44:48,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a5ba742b83c646238f46e756ed942c4a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/a5ba742b83c646238f46e756ed942c4a 2024-12-06T14:44:48,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/db28c51c9aa648199639b00ca10c2b9d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/db28c51c9aa648199639b00ca10c2b9d 2024-12-06T14:44:48,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a6389fa8057472bb8788a32d0bd6615 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/7a6389fa8057472bb8788a32d0bd6615 2024-12-06T14:44:48,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/baa24a1a07574f8f9fa28d69e9acb7f2 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/baa24a1a07574f8f9fa28d69e9acb7f2 2024-12-06T14:44:48,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cb7181e6e821469b99396a3f15813ae9, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5dab2d7e162441c5950f4821d5a0ee23, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/644911a9e62e4f2089eea361975f350d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5c13cf4c740d4ffd8b9e00b0abfdeb53, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6fdfadfcc6d2437492bf446fb45fdba5, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac32369681bf43e99d4968616fb17131, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/e6195bb074cc4b80ab73c40dee96c018, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/24de46c7fac94cb6a83ad6706e43c40a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf88c86deca640b1a10a2b04e666d2c0, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/2168fb45ff4048278030d75c5abcc699, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/aad09bdce7f34a66956fb0a92e302272, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/93a084cb40564ba2939c2e20eebd8535, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d4db40032f5c40d7b9fa86659d4dcf7c, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6249e492e1684fb884fb779b8844227d, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3425569c8db749a098ae71768a6b2e84, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf397c4e92044ca8a353fa38702c8497, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b44bf668837146b5975281e805a32ffb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/fc4064859eb04151a409c4371ddbecc1, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/9110842258964f53a0d95da554e13fc6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b9d910e71ec24989b79480a412e066fc, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac367bd956d748c38b60c53c1adf2f9a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6779f75185e94277bb18468e2be35bcb, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/7ebe6c265d3a471d9f81dd7efbb38f65, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/20b701dc1c7349b8adbeb31765d2efd6, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/12e3fc2cec8e401782fb61df69ad174a, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf0665df62c94d73861678cd705aff5f, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/012c96eab62645b2a54955ae3783d745, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d35bb730f4454871905e98fc119af185] to archive 2024-12-06T14:44:48,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T14:44:48,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cb7181e6e821469b99396a3f15813ae9 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cb7181e6e821469b99396a3f15813ae9 2024-12-06T14:44:48,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5dab2d7e162441c5950f4821d5a0ee23 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5dab2d7e162441c5950f4821d5a0ee23 2024-12-06T14:44:48,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/644911a9e62e4f2089eea361975f350d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/644911a9e62e4f2089eea361975f350d 2024-12-06T14:44:48,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5c13cf4c740d4ffd8b9e00b0abfdeb53 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/5c13cf4c740d4ffd8b9e00b0abfdeb53 2024-12-06T14:44:48,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6fdfadfcc6d2437492bf446fb45fdba5 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6fdfadfcc6d2437492bf446fb45fdba5 2024-12-06T14:44:48,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac32369681bf43e99d4968616fb17131 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac32369681bf43e99d4968616fb17131 2024-12-06T14:44:48,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/e6195bb074cc4b80ab73c40dee96c018 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/e6195bb074cc4b80ab73c40dee96c018 2024-12-06T14:44:48,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/24de46c7fac94cb6a83ad6706e43c40a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/24de46c7fac94cb6a83ad6706e43c40a 2024-12-06T14:44:48,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf88c86deca640b1a10a2b04e666d2c0 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf88c86deca640b1a10a2b04e666d2c0 2024-12-06T14:44:48,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/2168fb45ff4048278030d75c5abcc699 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/2168fb45ff4048278030d75c5abcc699 2024-12-06T14:44:48,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/aad09bdce7f34a66956fb0a92e302272 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/aad09bdce7f34a66956fb0a92e302272 2024-12-06T14:44:48,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/93a084cb40564ba2939c2e20eebd8535 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/93a084cb40564ba2939c2e20eebd8535 2024-12-06T14:44:48,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d4db40032f5c40d7b9fa86659d4dcf7c to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d4db40032f5c40d7b9fa86659d4dcf7c 2024-12-06T14:44:48,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6249e492e1684fb884fb779b8844227d to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6249e492e1684fb884fb779b8844227d 2024-12-06T14:44:48,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3425569c8db749a098ae71768a6b2e84 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3425569c8db749a098ae71768a6b2e84 2024-12-06T14:44:48,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf397c4e92044ca8a353fa38702c8497 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf397c4e92044ca8a353fa38702c8497 2024-12-06T14:44:48,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b44bf668837146b5975281e805a32ffb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b44bf668837146b5975281e805a32ffb 2024-12-06T14:44:48,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/fc4064859eb04151a409c4371ddbecc1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/fc4064859eb04151a409c4371ddbecc1 2024-12-06T14:44:48,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/9110842258964f53a0d95da554e13fc6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/9110842258964f53a0d95da554e13fc6 2024-12-06T14:44:48,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b9d910e71ec24989b79480a412e066fc to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/b9d910e71ec24989b79480a412e066fc 2024-12-06T14:44:48,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac367bd956d748c38b60c53c1adf2f9a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/ac367bd956d748c38b60c53c1adf2f9a 2024-12-06T14:44:48,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6779f75185e94277bb18468e2be35bcb to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/6779f75185e94277bb18468e2be35bcb 2024-12-06T14:44:48,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/7ebe6c265d3a471d9f81dd7efbb38f65 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/7ebe6c265d3a471d9f81dd7efbb38f65 2024-12-06T14:44:48,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/20b701dc1c7349b8adbeb31765d2efd6 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/20b701dc1c7349b8adbeb31765d2efd6 2024-12-06T14:44:48,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/12e3fc2cec8e401782fb61df69ad174a to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/12e3fc2cec8e401782fb61df69ad174a 2024-12-06T14:44:48,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf0665df62c94d73861678cd705aff5f to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/cf0665df62c94d73861678cd705aff5f 2024-12-06T14:44:48,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/012c96eab62645b2a54955ae3783d745 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/012c96eab62645b2a54955ae3783d745 2024-12-06T14:44:48,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d35bb730f4454871905e98fc119af185 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/d35bb730f4454871905e98fc119af185 2024-12-06T14:44:48,260 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=4 2024-12-06T14:44:48,260 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. 2024-12-06T14:44:48,261 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 3623e1fd493fe297ada6ccd9ac6123ae: Waiting for close lock at 1733496286483Running coprocessor pre-close hooks at 1733496286483Disabling compacts and flushes for region at 1733496286483Disabling writes for close at 1733496286483Obtaining lock to block concurrent updates at 1733496286483Preparing flush snapshotting stores in 3623e1fd493fe297ada6ccd9ac6123ae at 1733496286483Finished memstore snapshotting TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733496286483Flushing stores of TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae. at 1733496286484 (+1 ms)Flushing 3623e1fd493fe297ada6ccd9ac6123ae/A: creating writer at 1733496286484Flushing mob file 3623e1fd493fe297ada6ccd9ac6123ae/A: appending metadata at 1733496286489 (+5 ms)Flushing mob file 3623e1fd493fe297ada6ccd9ac6123ae/A: closing flushed file at 1733496286489Flushing 3623e1fd493fe297ada6ccd9ac6123ae/A: appending metadata at 1733496286905 (+416 ms)Flushing 3623e1fd493fe297ada6ccd9ac6123ae/A: closing flushed file at 1733496286905Flushing 3623e1fd493fe297ada6ccd9ac6123ae/B: creating writer at 1733496287323 (+418 ms)Flushing 3623e1fd493fe297ada6ccd9ac6123ae/B: appending metadata at 1733496287325 (+2 ms)Flushing 3623e1fd493fe297ada6ccd9ac6123ae/B: closing flushed file at 1733496287325Flushing 3623e1fd493fe297ada6ccd9ac6123ae/C: creating writer at 1733496287740 (+415 ms)Flushing 3623e1fd493fe297ada6ccd9ac6123ae/C: appending metadata at 1733496287744 (+4 ms)Flushing 3623e1fd493fe297ada6ccd9ac6123ae/C: closing flushed file at 1733496287744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fff7089: reopening flushed file at 1733496288159 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403e286: reopening flushed file at 1733496288164 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@600569c9: reopening flushed file at 1733496288168 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3623e1fd493fe297ada6ccd9ac6123ae in 1689ms, sequenceid=403, compaction requested=false at 1733496288173 (+5 ms)Writing region close event to WAL at 1733496288258 (+85 ms)Running coprocessor post-close hooks at 1733496288260 (+2 ms)Closed at 1733496288260 2024-12-06T14:44:48,262 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=3623e1fd493fe297ada6ccd9ac6123ae, regionState=CLOSED 2024-12-06T14:44:48,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 because future has completed 2024-12-06T14:44:48,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-06T14:44:48,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 3623e1fd493fe297ada6ccd9ac6123ae, server=bb7ea3e46d9c,35831,1733496100253 in 1.9330 sec 2024-12-06T14:44:48,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-06T14:44:48,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3623e1fd493fe297ada6ccd9ac6123ae, UNASSIGN in 1.9370 sec 2024-12-06T14:44:48,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-06T14:44:48,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9390 sec 2024-12-06T14:44:48,268 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496288268"}]},"ts":"1733496288268"} 2024-12-06T14:44:48,269 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T14:44:48,269 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T14:44:48,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9480 sec 2024-12-06T14:44:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T14:44:48,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:48,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T14:44:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:48,474 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-06T14:44:48,476 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:48,479 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,482 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/recovered.edits] 2024-12-06T14:44:48,485 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b96431753b0c43c2a2598fab11425b47 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/b96431753b0c43c2a2598fab11425b47 2024-12-06T14:44:48,486 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/c5a66b27d15642b983ede4d49eb5a684 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/A/c5a66b27d15642b983ede4d49eb5a684 2024-12-06T14:44:48,489 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4ecdd1ddf04e423789bd3ece5ef8a36b to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/4ecdd1ddf04e423789bd3ece5ef8a36b 2024-12-06T14:44:48,490 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9db25e9eaf94498f836750b27b726e52 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/B/9db25e9eaf94498f836750b27b726e52 2024-12-06T14:44:48,493 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3ca6e7fe703c453580ee8a4bee40c790 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/3ca6e7fe703c453580ee8a4bee40c790 2024-12-06T14:44:48,494 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/55dbcdf114c9441f9d6cbf832f03fcb1 to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/C/55dbcdf114c9441f9d6cbf832f03fcb1 2024-12-06T14:44:48,496 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/recovered.edits/406.seqid to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae/recovered.edits/406.seqid 2024-12-06T14:44:48,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/default/TestAcidGuarantees/3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,497 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T14:44:48,497 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T14:44:48,498 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T14:44:48,501 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206049e148da147456490b04070b53f6454_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206049e148da147456490b04070b53f6454_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,502 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061083e52c0b0e43c2b53c05a9a01ccef6_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061083e52c0b0e43c2b53c05a9a01ccef6_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,504 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061b18d7589fec4dc981b5cb830ca6e010_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061b18d7589fec4dc981b5cb830ca6e010_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,505 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206202b0a79e27145e9a205b93df812b8d0_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206202b0a79e27145e9a205b93df812b8d0_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,506 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063f2453764abb479ba5bb9d78408bdfdf_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063f2453764abb479ba5bb9d78408bdfdf_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,507 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064dece1a8f9cf439bb28ad709b2c7aed2_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064dece1a8f9cf439bb28ad709b2c7aed2_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,508 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065b77cfa100dd4e9a83dd5b59c5482e1e_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065b77cfa100dd4e9a83dd5b59c5482e1e_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,509 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120673ddc50911ad4be58afabac160e04353_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120673ddc50911ad4be58afabac160e04353_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,510 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120681570d25c0be4ea6bc06cd17072bcf39_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120681570d25c0be4ea6bc06cd17072bcf39_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,511 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206949a956ded9c4bc0984eaa203d7614be_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206949a956ded9c4bc0984eaa203d7614be_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,511 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069914105a2bcb414fac25f590419fdf4b_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069914105a2bcb414fac25f590419fdf4b_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,512 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206998ae1c549874fe1850588332dd2373c_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206998ae1c549874fe1850588332dd2373c_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,513 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a210629e265a4968b394406d8f47b5ec_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a210629e265a4968b394406d8f47b5ec_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,514 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a6d025e96b7746b8b414e4312a67d508_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a6d025e96b7746b8b414e4312a67d508_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,515 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a7804ce35d18470899eb5d63720ca0c6_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a7804ce35d18470899eb5d63720ca0c6_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,516 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c4e8b8db203e4c62ad4ac96cbddec8b6_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c4e8b8db203e4c62ad4ac96cbddec8b6_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,516 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c64abc4796224ba7b30ba21ac1286b27_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c64abc4796224ba7b30ba21ac1286b27_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,517 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dca718031929434bbffe72a0e2a3d05f_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dca718031929434bbffe72a0e2a3d05f_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,518 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dd84621a57e54369849ddf5bff313774_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dd84621a57e54369849ddf5bff313774_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,519 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e1255627cba3477e85f2142405585452_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e1255627cba3477e85f2142405585452_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,520 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f434195316a74f269a823cb26f948a70_3623e1fd493fe297ada6ccd9ac6123ae to hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f434195316a74f269a823cb26f948a70_3623e1fd493fe297ada6ccd9ac6123ae 2024-12-06T14:44:48,520 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T14:44:48,522 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:48,525 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T14:44:48,527 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T14:44:48,528 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:48,528 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T14:44:48,528 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733496288528"}]},"ts":"9223372036854775807"} 2024-12-06T14:44:48,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T14:44:48,530 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3623e1fd493fe297ada6ccd9ac6123ae, NAME => 'TestAcidGuarantees,,1733496255266.3623e1fd493fe297ada6ccd9ac6123ae.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T14:44:48,530 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T14:44:48,530 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733496288530"}]},"ts":"9223372036854775807"} 2024-12-06T14:44:48,533 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T14:44:48,533 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T14:44:48,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-12-06T14:44:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-06T14:44:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T14:44:48,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T14:44:48,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 248), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 554), ProcessCount=11 (was 11), AvailableMemoryMB=4020 (was 6682) 2024-12-06T14:44:48,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T14:44:48,593 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:44:48,593 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:44:48,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:48,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:48,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:44:48,594 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T14:44:48,594 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-12-06T14:44:48,595 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bb7ea3e46d9c,38707,1733496099493 2024-12-06T14:44:48,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:44:48,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:44:48,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:44:48,656 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T14:44:48,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:44:48,656 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:44:48,657 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:44:48,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:48,658 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,35831,1733496100253' ***** 2024-12-06T14:44:48,658 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T14:44:48,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:44:48,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:44:48,659 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T14:44:48,659 INFO [RS:0;bb7ea3e46d9c:35831 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T14:44:48,659 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T14:44:48,660 INFO [RS:0;bb7ea3e46d9c:35831 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T14:44:48,660 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:48,660 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:44:48,660 INFO [RS:0;bb7ea3e46d9c:35831 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bb7ea3e46d9c:35831. 2024-12-06T14:44:48,660 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:44:48,660 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:48,660 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T14:44:48,661 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T14:44:48,661 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T14:44:48,661 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T14:44:48,662 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T14:44:48,662 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-06T14:44:48,662 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:48,662 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:44:48,662 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:44:48,662 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:44:48,662 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:44:48,662 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:44:48,663 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-06T14:44:48,681 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/info/400c784d5fba49c7987307e546922faf is 95, key is TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843./info:/1733496132946/DeleteFamily/seqid=0 2024-12-06T14:44:48,684 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:44:48,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742468_1644 (size=6514) 2024-12-06T14:44:48,862 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:49,063 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:49,085 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/info/400c784d5fba49c7987307e546922faf 2024-12-06T14:44:49,111 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/ns/2e29cb9d76c0463288bbc9126ec2e981 is 93, key is TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843./ns:/1733496132910/DeleteFamily/seqid=0 2024-12-06T14:44:49,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742469_1645 (size=5906) 2024-12-06T14:44:49,263 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:49,328 INFO [regionserver/bb7ea3e46d9c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T14:44:49,328 INFO [regionserver/bb7ea3e46d9c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T14:44:49,463 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:49,515 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/ns/2e29cb9d76c0463288bbc9126ec2e981 2024-12-06T14:44:49,530 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/rep_barrier/6b5f31f508d447d4b5b2708ac2a5897f is 102, key is TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843./rep_barrier:/1733496132910/DeleteFamily/seqid=0 2024-12-06T14:44:49,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742470_1646 (size=6025) 2024-12-06T14:44:49,664 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:49,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T14:44:49,864 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:49,933 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/rep_barrier/6b5f31f508d447d4b5b2708ac2a5897f 2024-12-06T14:44:49,956 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/table/29b9e439c94a4d039edeb4ea6d3723e6 is 96, key is TestAcidGuarantees,,1733496104971.709a255a23a9b4b1363f2ec84dd61843./table:/1733496132910/DeleteFamily/seqid=0 2024-12-06T14:44:49,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742471_1647 (size=5936) 2024-12-06T14:44:50,064 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:50,264 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:44:50,359 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/table/29b9e439c94a4d039edeb4ea6d3723e6 2024-12-06T14:44:50,362 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/info/400c784d5fba49c7987307e546922faf as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/info/400c784d5fba49c7987307e546922faf 2024-12-06T14:44:50,365 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/info/400c784d5fba49c7987307e546922faf, entries=12, sequenceid=90, filesize=6.4 K 2024-12-06T14:44:50,366 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/ns/2e29cb9d76c0463288bbc9126ec2e981 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/ns/2e29cb9d76c0463288bbc9126ec2e981 2024-12-06T14:44:50,369 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/ns/2e29cb9d76c0463288bbc9126ec2e981, entries=8, sequenceid=90, filesize=5.8 K 2024-12-06T14:44:50,370 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/rep_barrier/6b5f31f508d447d4b5b2708ac2a5897f as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/rep_barrier/6b5f31f508d447d4b5b2708ac2a5897f 2024-12-06T14:44:50,373 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/rep_barrier/6b5f31f508d447d4b5b2708ac2a5897f, entries=6, sequenceid=90, filesize=5.9 K 2024-12-06T14:44:50,374 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/.tmp/table/29b9e439c94a4d039edeb4ea6d3723e6 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/table/29b9e439c94a4d039edeb4ea6d3723e6 2024-12-06T14:44:50,376 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/table/29b9e439c94a4d039edeb4ea6d3723e6, entries=7, sequenceid=90, filesize=5.8 K 2024-12-06T14:44:50,377 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false 2024-12-06T14:44:50,384 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-06T14:44:50,385 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T14:44:50,385 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:44:50,385 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496288662Running coprocessor pre-close hooks at 1733496288662Disabling compacts and flushes for region at 1733496288662Disabling writes for close at 1733496288662Obtaining lock to block concurrent updates at 1733496288663 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733496288663Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733496288663Flushing stores of hbase:meta,,1.1588230740 at 1733496288664 (+1 ms)Flushing 1588230740/info: creating writer at 1733496288664Flushing 1588230740/info: appending metadata at 1733496288680 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733496288680Flushing 1588230740/ns: creating writer at 1733496289093 (+413 ms)Flushing 1588230740/ns: appending metadata at 1733496289111 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733496289111Flushing 1588230740/rep_barrier: creating writer at 1733496289518 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733496289530 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733496289530Flushing 1588230740/table: creating writer at 1733496289937 (+407 ms)Flushing 1588230740/table: appending metadata at 1733496289955 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733496289955Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6052ab57: reopening flushed file at 1733496290362 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55dfae42: reopening flushed file at 1733496290366 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a9eca15: reopening flushed file at 1733496290369 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5569ee4d: reopening flushed file at 1733496290373 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false at 1733496290377 (+4 ms)Writing region close event to WAL at 1733496290381 (+4 ms)Running coprocessor post-close hooks at 1733496290385 (+4 ms)Closed at 1733496290385 2024-12-06T14:44:50,385 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T14:44:50,465 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,35831,1733496100253; all regions closed. 2024-12-06T14:44:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741834_1010 (size=25344) 2024-12-06T14:44:50,471 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/oldWALs 2024-12-06T14:44:50,471 INFO [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C35831%2C1733496100253.meta:.meta(num 1733496104478) 2024-12-06T14:44:50,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741832_1008 (size=21485313) 2024-12-06T14:44:50,475 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/oldWALs 2024-12-06T14:44:50,475 INFO [RS:0;bb7ea3e46d9c:35831 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C35831%2C1733496100253:(num 1733496103474) 2024-12-06T14:44:50,475 DEBUG [RS:0;bb7ea3e46d9c:35831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:44:50,475 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:44:50,475 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:44:50,476 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T14:44:50,476 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:44:50,476 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:44:50,477 INFO [RS:0;bb7ea3e46d9c:35831 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35831 2024-12-06T14:44:50,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,35831,1733496100253 2024-12-06T14:44:50,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:44:50,550 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:44:50,550 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,35831,1733496100253] 2024-12-06T14:44:50,571 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,35831,1733496100253 already deleted, retry=false 2024-12-06T14:44:50,571 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bb7ea3e46d9c,35831,1733496100253 expired; onlineServers=0 2024-12-06T14:44:50,571 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bb7ea3e46d9c,38707,1733496099493' ***** 2024-12-06T14:44:50,571 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T14:44:50,571 INFO [M:0;bb7ea3e46d9c:38707 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:44:50,571 INFO [M:0;bb7ea3e46d9c:38707 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:44:50,571 DEBUG [M:0;bb7ea3e46d9c:38707 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T14:44:50,572 DEBUG [M:0;bb7ea3e46d9c:38707 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T14:44:50,572 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T14:44:50,572 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496103166 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496103166,5,FailOnTimeoutGroup] 2024-12-06T14:44:50,572 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496103166 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496103166,5,FailOnTimeoutGroup] 2024-12-06T14:44:50,572 INFO [M:0;bb7ea3e46d9c:38707 {}] hbase.ChoreService(370): Chore service for: master/bb7ea3e46d9c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T14:44:50,572 INFO [M:0;bb7ea3e46d9c:38707 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:44:50,572 DEBUG [M:0;bb7ea3e46d9c:38707 {}] master.HMaster(1795): Stopping service threads 2024-12-06T14:44:50,572 INFO [M:0;bb7ea3e46d9c:38707 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T14:44:50,572 INFO [M:0;bb7ea3e46d9c:38707 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T14:44:50,572 ERROR [M:0;bb7ea3e46d9c:38707 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-06T14:44:50,573 INFO [M:0;bb7ea3e46d9c:38707 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T14:44:50,573 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T14:44:50,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T14:44:50,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:44:50,592 DEBUG [M:0;bb7ea3e46d9c:38707 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-06T14:44:50,592 DEBUG [M:0;bb7ea3e46d9c:38707 {}] master.ActiveMasterManager(353): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-06T14:44:50,592 INFO [M:0;bb7ea3e46d9c:38707 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/.lastflushedseqids 2024-12-06T14:44:50,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742472_1648 (size=72) 2024-12-06T14:44:50,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:44:50,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35831-0x101ab174acf0001, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:44:50,661 INFO [RS:0;bb7ea3e46d9c:35831 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:44:50,661 INFO [RS:0;bb7ea3e46d9c:35831 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,35831,1733496100253; zookeeper connection closed. 2024-12-06T14:44:50,662 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16c6478a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16c6478a 2024-12-06T14:44:50,662 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-06T14:44:51,004 INFO [M:0;bb7ea3e46d9c:38707 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T14:44:51,005 INFO [M:0;bb7ea3e46d9c:38707 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T14:44:51,005 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T14:44:51,005 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:44:51,005 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:44:51,005 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T14:44:51,005 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:44:51,005 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.83 KB heapSize=949.60 KB 2024-12-06T14:44:51,024 DEBUG [M:0;bb7ea3e46d9c:38707 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ae69bea36b1d4529b00f5e11c496ada1 is 82, key is hbase:meta,,1/info:regioninfo/1733496104559/Put/seqid=0 2024-12-06T14:44:51,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742473_1649 (size=5672) 2024-12-06T14:44:51,428 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2161 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ae69bea36b1d4529b00f5e11c496ada1 2024-12-06T14:44:51,448 DEBUG [M:0;bb7ea3e46d9c:38707 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9a6e97281244417ad220e2ac7154188 is 2479, key is \x00\x00\x00\x00\x00\x00\x00V/proc:d/1733496194775/Put/seqid=0 2024-12-06T14:44:51,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742474_1650 (size=44651) 2024-12-06T14:44:51,854 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=774.28 KB at sequenceid=2161 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9a6e97281244417ad220e2ac7154188 2024-12-06T14:44:51,861 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f9a6e97281244417ad220e2ac7154188 2024-12-06T14:44:51,879 DEBUG [M:0;bb7ea3e46d9c:38707 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/da0d1d75d63d4d74b090349fccb45077 is 69, key is bb7ea3e46d9c,35831,1733496100253/rs:state/1733496103239/Put/seqid=0 2024-12-06T14:44:51,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073742475_1651 (size=5156) 2024-12-06T14:44:52,282 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2161 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/da0d1d75d63d4d74b090349fccb45077 2024-12-06T14:44:52,287 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ae69bea36b1d4529b00f5e11c496ada1 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ae69bea36b1d4529b00f5e11c496ada1 2024-12-06T14:44:52,291 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ae69bea36b1d4529b00f5e11c496ada1, entries=8, sequenceid=2161, filesize=5.5 K 2024-12-06T14:44:52,291 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9a6e97281244417ad220e2ac7154188 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f9a6e97281244417ad220e2ac7154188 2024-12-06T14:44:52,295 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f9a6e97281244417ad220e2ac7154188 2024-12-06T14:44:52,295 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f9a6e97281244417ad220e2ac7154188, entries=166, sequenceid=2161, filesize=43.6 K 2024-12-06T14:44:52,295 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/da0d1d75d63d4d74b090349fccb45077 as hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/da0d1d75d63d4d74b090349fccb45077 2024-12-06T14:44:52,298 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/c7305de9-5a5d-03ec-82a7-48359d3a06e5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/da0d1d75d63d4d74b090349fccb45077, entries=1, sequenceid=2161, filesize=5.0 K 2024-12-06T14:44:52,299 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.83 KB/793428, heapSize ~949.30 KB/972088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=2161, compaction requested=false 2024-12-06T14:44:52,300 INFO [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:44:52,300 DEBUG [M:0;bb7ea3e46d9c:38707 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733496291005Disabling compacts and flushes for region at 1733496291005Disabling writes for close at 1733496291005Obtaining lock to block concurrent updates at 1733496291005Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733496291005Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=793428, getHeapSize=972328, getOffHeapSize=0, getCellsCount=2352 at 1733496291006 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733496291007 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733496291007Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733496291024 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733496291024Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733496291431 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733496291448 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733496291448Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733496291861 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733496291878 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733496291879 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18fa738e: reopening flushed file at 1733496292286 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@512c77e3: reopening flushed file at 1733496292291 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24265e30: reopening flushed file at 1733496292295 (+4 ms)Finished flush of dataSize ~774.83 KB/793428, heapSize ~949.30 KB/972088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=2161, compaction requested=false at 1733496292299 (+4 ms)Writing region close event to WAL at 1733496292300 (+1 ms)Closed at 1733496292300 2024-12-06T14:44:52,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35987 is added to blk_1073741830_1006 (size=935076) 2024-12-06T14:44:52,303 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:44:52,303 INFO [M:0;bb7ea3e46d9c:38707 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T14:44:52,303 INFO [M:0;bb7ea3e46d9c:38707 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38707 2024-12-06T14:44:52,303 INFO [M:0;bb7ea3e46d9c:38707 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:44:52,418 INFO [M:0;bb7ea3e46d9c:38707 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:44:52,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:44:52,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38707-0x101ab174acf0000, quorum=127.0.0.1:63690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:44:52,422 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T14:44:52,424 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T14:44:52,424 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T14:44:52,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T14:44:52,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/hadoop.log.dir/,STOPPED} 2024-12-06T14:44:52,427 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T14:44:52,427 WARN [BP-1636499811-172.17.0.2-1733496095640 heartbeating to localhost/127.0.0.1:35677 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T14:44:52,427 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T14:44:52,427 WARN [BP-1636499811-172.17.0.2-1733496095640 heartbeating to localhost/127.0.0.1:35677 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1636499811-172.17.0.2-1733496095640 (Datanode Uuid 91e1b61b-e56e-40a9-ae1f-956ec261e67b) service to localhost/127.0.0.1:35677 2024-12-06T14:44:52,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/data/data1/current/BP-1636499811-172.17.0.2-1733496095640 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T14:44:52,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/cluster_c1e5597b-1b21-bbc8-d4fd-8a874b86bb93/data/data2/current/BP-1636499811-172.17.0.2-1733496095640 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T14:44:52,430 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T14:44:52,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T14:44:52,437 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T14:44:52,437 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T14:44:52,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T14:44:52,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/04e9d663-5873-f2ae-5b03-99c16bb29520/hadoop.log.dir/,STOPPED} 2024-12-06T14:44:52,453 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T14:44:52,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down